blob: 44a9dcb57609b503bc60c9f9a8b56a4c5e949bc2 [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
Willy Tarreau62405a22014-12-23 13:51:28 +01002 * include/common/memory.h
3 * Memory management definitions..
4 *
5 * Copyright (C) 2000-2014 Willy Tarreau - w@1wt.eu
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation, version 2.1
10 * exclusively.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
Willy Tarreaubaaee002006-06-26 02:48:02 +020021
Willy Tarreau2dd0d472006-06-29 17:53:05 +020022#ifndef _COMMON_MEMORY_H
23#define _COMMON_MEMORY_H
Willy Tarreaubaaee002006-06-26 02:48:02 +020024
Willy Tarreau158fa752017-11-22 15:47:29 +010025#include <sys/mman.h>
26
Willy Tarreaubaaee002006-06-26 02:48:02 +020027#include <stdlib.h>
Willy Tarreaue430e772014-12-23 14:13:16 +010028#include <string.h>
Willy Tarreaua1bd1fa2019-03-29 17:26:33 +010029#include <inttypes.h>
Willy Tarreaua7280a12018-11-26 19:41:40 +010030#include <unistd.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020031
Willy Tarreau2dd0d472006-06-29 17:53:05 +020032#include <common/config.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020033#include <common/mini-clist.h>
Christopher Fauletb349e482017-08-29 09:52:38 +020034#include <common/hathreads.h>
Willy Tarreau7107c8b2018-11-26 11:44:35 +010035#include <common/initcall.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020036
Willy Tarreaua84dcb82015-10-28 12:04:02 +010037#ifndef DEBUG_DONT_SHARE_POOLS
Willy Tarreau50e608d2007-05-13 18:26:08 +020038#define MEM_F_SHARED 0x1
Willy Tarreaua84dcb82015-10-28 12:04:02 +010039#else
40#define MEM_F_SHARED 0
41#endif
Willy Tarreau581bf812016-01-25 02:19:13 +010042#define MEM_F_EXACT 0x2
Willy Tarreau50e608d2007-05-13 18:26:08 +020043
Willy Tarreauac421112015-10-28 15:09:29 +010044/* reserve an extra void* at the end of a pool for linking */
45#ifdef DEBUG_MEMORY_POOLS
46#define POOL_EXTRA (sizeof(void *))
47#define POOL_LINK(pool, item) (void **)(((char *)item) + (pool->size))
48#else
49#define POOL_EXTRA (0)
50#define POOL_LINK(pool, item) ((void **)(item))
51#endif
52
Willy Tarreau0a93b642018-10-16 07:58:39 +020053#define MAX_BASE_POOLS 32
54
Willy Tarreaue18db9e2018-10-16 10:28:54 +020055struct pool_cache_head {
56 struct list list; /* head of objects in this pool */
57 size_t size; /* size of an object */
58 unsigned int count; /* number of objects in this pool */
59};
60
61struct pool_cache_item {
62 struct list by_pool; /* link to objects in this pool */
63 struct list by_lru; /* link to objects by LRU order */
64};
65
Willy Tarreau7f0165e2018-11-26 17:09:46 +010066extern struct pool_cache_head pool_cache[][MAX_BASE_POOLS];
Willy Tarreaue18db9e2018-10-16 10:28:54 +020067extern THREAD_LOCAL size_t pool_cache_bytes; /* total cache size */
68extern THREAD_LOCAL size_t pool_cache_count; /* #cache objects */
69
Willy Tarreauf161d0f2018-02-22 14:05:55 +010070#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +010071struct pool_free_list {
72 void **free_list;
73 uintptr_t seq;
74};
75#endif
76
Willy Tarreau50e608d2007-05-13 18:26:08 +020077struct pool_head {
Willy Tarreau1ca1b702017-11-26 10:50:36 +010078 void **free_list;
Willy Tarreauf161d0f2018-02-22 14:05:55 +010079#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +010080 uintptr_t seq;
Olivier Houchard899fb8a2020-03-18 15:48:29 +010081 HA_SPINLOCK_T flush_lock;
Olivier Houchardcf975d42018-01-24 18:38:31 +010082#else
83 __decl_hathreads(HA_SPINLOCK_T lock); /* the spin lock */
84#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +020085 unsigned int used; /* how many chunks are currently in use */
86 unsigned int allocated; /* how many chunks have been allocated */
87 unsigned int limit; /* hard limit on the number of chunks */
88 unsigned int minavail; /* how many chunks are expected to be used */
89 unsigned int size; /* chunk size */
90 unsigned int flags; /* MEM_F_* */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020091 unsigned int users; /* number of pools sharing this zone */
Willy Tarreau58102cf2015-10-28 16:24:21 +010092 unsigned int failed; /* failed allocations */
Olivier Houchardcf975d42018-01-24 18:38:31 +010093 struct list list; /* list of all known pools */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020094 char name[12]; /* name of the pool */
Willy Tarreau1ca1b702017-11-26 10:50:36 +010095} __attribute__((aligned(64)));
Willy Tarreau50e608d2007-05-13 18:26:08 +020096
Willy Tarreau0a93b642018-10-16 07:58:39 +020097
98extern struct pool_head pool_base_start[MAX_BASE_POOLS];
99extern unsigned int pool_base_count;
100
Willy Tarreau067ac9f2015-10-08 14:12:13 +0200101/* poison each newly allocated area with this byte if >= 0 */
102extern int mem_poison_byte;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200103
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100104/* Allocates new entries for pool <pool> until there are at least <avail> + 1
105 * available, then returns the last one for immediate use, so that at least
106 * <avail> are left available in the pool upon return. NULL is returned if the
107 * last entry could not be allocated. It's important to note that at least one
108 * allocation is always performed even if there are enough entries in the pool.
109 * A call to the garbage collector is performed at most once in case malloc()
110 * returns an error, before returning NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200111 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200112void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100113void *pool_refill_alloc(struct pool_head *pool, unsigned int avail);
Willy Tarreau50e608d2007-05-13 18:26:08 +0200114
115/* Try to find an existing shared pool with the same characteristics and
116 * returns it, otherwise creates this one. NULL is returned if no memory
117 * is available for a new creation.
118 */
119struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags);
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100120void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size);
121
122/* This registers a call to create_pool_callback(ptr, name, size) */
123#define REGISTER_POOL(ptr, name, size) \
124 INITCALL3(STG_POOL, create_pool_callback, (ptr), (name), (size))
125
126/* This macro declares a pool head <ptr> and registers its creation */
127#define DECLARE_POOL(ptr, name, size) \
128 struct pool_head *(ptr) = NULL; \
129 REGISTER_POOL(&ptr, name, size)
130
131/* This macro declares a static pool head <ptr> and registers its creation */
132#define DECLARE_STATIC_POOL(ptr, name, size) \
133 static struct pool_head *(ptr); \
134 REGISTER_POOL(&ptr, name, size)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200135
136/* Dump statistics on pools usage.
137 */
Willy Tarreau12833bb2014-01-28 16:49:56 +0100138void dump_pools_to_trash();
Willy Tarreau50e608d2007-05-13 18:26:08 +0200139void dump_pools(void);
Willy Tarreau58102cf2015-10-28 16:24:21 +0100140int pool_total_failures();
141unsigned long pool_total_allocated();
142unsigned long pool_total_used();
Willy Tarreau50e608d2007-05-13 18:26:08 +0200143
144/*
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200145 * This function frees whatever can be freed in pool <pool>.
146 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100147void pool_flush(struct pool_head *pool);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200148
149/*
150 * This function frees whatever can be freed in all pools, but respecting
151 * the minimum thresholds imposed by owners.
Christopher Fauletb349e482017-08-29 09:52:38 +0200152 *
Willy Tarreaubafbe012017-11-24 17:34:44 +0100153 * <pool_ctx> is used when pool_gc is called to release resources to allocate
Christopher Fauletb349e482017-08-29 09:52:38 +0200154 * an element in __pool_refill_alloc. It is important because <pool_ctx> is
155 * already locked, so we need to skip the lock here.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200156 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100157void pool_gc(struct pool_head *pool_ctx);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200158
159/*
160 * This function destroys a pull by freeing it completely.
161 * This should be called only under extreme circumstances.
162 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100163void *pool_destroy(struct pool_head *pool);
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100164void pool_destroy_all();
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200165
Willy Tarreau0a93b642018-10-16 07:58:39 +0200166/* returns the pool index for pool <pool>, or -1 if this pool has no index */
167static inline ssize_t pool_get_index(const struct pool_head *pool)
168{
169 size_t idx;
170
171 idx = pool - pool_base_start;
172 if (idx >= MAX_BASE_POOLS)
173 return -1;
174 return idx;
175}
176
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100177#ifdef CONFIG_HAP_LOCKLESS_POOLS
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200178
179/* Tries to retrieve an object from the local pool cache corresponding to pool
180 * <pool>. Returns NULL if none is available.
181 */
182static inline void *__pool_get_from_cache(struct pool_head *pool)
183{
184 ssize_t idx = pool_get_index(pool);
185 struct pool_cache_item *item;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100186 struct pool_cache_head *ph;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200187
188 /* pool not in cache */
189 if (idx < 0)
190 return NULL;
191
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100192 ph = &pool_cache[tid][idx];
193 if (LIST_ISEMPTY(&ph->list))
194 return NULL; // empty
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200195
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100196 item = LIST_NEXT(&ph->list, typeof(item), by_pool);
197 ph->count--;
198 pool_cache_bytes -= ph->size;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200199 pool_cache_count--;
200 LIST_DEL(&item->by_pool);
201 LIST_DEL(&item->by_lru);
Willy Tarreau8e9f4532018-10-28 20:09:12 +0100202#ifdef DEBUG_MEMORY_POOLS
203 /* keep track of where the element was allocated from */
204 *POOL_LINK(pool, item) = (void *)pool;
205#endif
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200206 return item;
207}
208
Olivier Houchardcf975d42018-01-24 18:38:31 +0100209/*
210 * Returns a pointer to type <type> taken from the pool <pool_type> if
211 * available, otherwise returns NULL. No malloc() is attempted, and poisonning
212 * is never performed. The purpose is to get the fastest possible allocation.
213 */
214static inline void *__pool_get_first(struct pool_head *pool)
215{
216 struct pool_free_list cmp, new;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200217 void *ret = __pool_get_from_cache(pool);
218
219 if (ret)
220 return ret;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100221
222 cmp.seq = pool->seq;
223 __ha_barrier_load();
224
225 cmp.free_list = pool->free_list;
226 do {
Olivier Houchard899fb8a2020-03-18 15:48:29 +0100227 if (cmp.free_list == NULL)
Olivier Houchardcf975d42018-01-24 18:38:31 +0100228 return NULL;
229 new.seq = cmp.seq + 1;
230 __ha_barrier_load();
231 new.free_list = *POOL_LINK(pool, cmp.free_list);
Willy Tarreau6a38b322019-05-11 18:04:24 +0200232 } while (HA_ATOMIC_DWCAS((void *)&pool->free_list, (void *)&cmp, (void *)&new) == 0);
Olivier Houchard20872762019-03-08 18:53:35 +0100233 __ha_barrier_atomic_store();
Tim Duesterhus05f6a432018-02-20 00:49:46 +0100234
Olivier Houchard20872762019-03-08 18:53:35 +0100235 _HA_ATOMIC_ADD(&pool->used, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100236#ifdef DEBUG_MEMORY_POOLS
237 /* keep track of where the element was allocated from */
238 *POOL_LINK(pool, cmp.free_list) = (void *)pool;
239#endif
240 return cmp.free_list;
241}
242
243static inline void *pool_get_first(struct pool_head *pool)
244{
245 void *ret;
246
247 ret = __pool_get_first(pool);
248 return ret;
249}
250/*
251 * Returns a pointer to type <type> taken from the pool <pool_type> or
252 * dynamically allocated. In the first case, <pool_type> is updated to point to
253 * the next element in the list. No memory poisonning is ever performed on the
254 * returned area.
255 */
256static inline void *pool_alloc_dirty(struct pool_head *pool)
257{
258 void *p;
259
260 if ((p = __pool_get_first(pool)) == NULL)
261 p = __pool_refill_alloc(pool, 0);
262 return p;
263}
264
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200265/*
Olivier Houchardcf975d42018-01-24 18:38:31 +0100266 * Returns a pointer to type <type> taken from the pool <pool_type> or
267 * dynamically allocated. In the first case, <pool_type> is updated to point to
268 * the next element in the list. Memory poisonning is performed if enabled.
269 */
270static inline void *pool_alloc(struct pool_head *pool)
271{
272 void *p;
273
274 p = pool_alloc_dirty(pool);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100275 if (p && mem_poison_byte >= 0) {
276 memset(p, mem_poison_byte, pool->size);
277 }
278
279 return p;
280}
281
Willy Tarreau146794d2018-10-16 08:55:15 +0200282/* Locklessly add item <ptr> to pool <pool>, then update the pool used count.
283 * Both the pool and the pointer must be valid. Use pool_free() for normal
284 * operations.
285 */
286static inline void __pool_free(struct pool_head *pool, void *ptr)
287{
Willy Tarreau7a6ad882018-10-20 17:37:38 +0200288 void **free_list = pool->free_list;
Willy Tarreau146794d2018-10-16 08:55:15 +0200289
290 do {
291 *POOL_LINK(pool, ptr) = (void *)free_list;
292 __ha_barrier_store();
Olivier Houchard20872762019-03-08 18:53:35 +0100293 } while (!_HA_ATOMIC_CAS(&pool->free_list, &free_list, ptr));
294 __ha_barrier_atomic_store();
295 _HA_ATOMIC_SUB(&pool->used, 1);
Willy Tarreau146794d2018-10-16 08:55:15 +0200296}
297
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200298/* frees an object to the local cache, possibly pushing oldest objects to the
299 * global pool.
300 */
301void __pool_put_to_cache(struct pool_head *pool, void *ptr, ssize_t idx);
302static inline void pool_put_to_cache(struct pool_head *pool, void *ptr)
303{
304 ssize_t idx = pool_get_index(pool);
305
306 /* pool not in cache or too many objects for this pool (more than
307 * half of the cache is used and this pool uses more than 1/8 of
308 * the cache size).
309 */
310 if (idx < 0 ||
311 (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4 &&
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100312 pool_cache[tid][idx].count >= 16 + pool_cache_count / 8)) {
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200313 __pool_free(pool, ptr);
314 return;
315 }
316 __pool_put_to_cache(pool, ptr, idx);
317}
318
Olivier Houchardcf975d42018-01-24 18:38:31 +0100319/*
320 * Puts a memory area back to the corresponding pool.
321 * Items are chained directly through a pointer that
322 * is written in the beginning of the memory area, so
323 * there's no need for any carrier cell. This implies
324 * that each memory area is at least as big as one
325 * pointer. Just like with the libc's free(), nothing
326 * is done if <ptr> is NULL.
327 */
328static inline void pool_free(struct pool_head *pool, void *ptr)
329{
330 if (likely(ptr != NULL)) {
Olivier Houchardcf975d42018-01-24 18:38:31 +0100331#ifdef DEBUG_MEMORY_POOLS
332 /* we'll get late corruption if we refill to the wrong pool or double-free */
333 if (*POOL_LINK(pool, ptr) != (void *)pool)
Willy Tarreaue4d42552020-03-14 11:08:16 +0100334 *DISGUISE((volatile int *)0) = 0;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100335#endif
Willy Tarreauda520352019-11-15 06:59:54 +0100336 if (mem_poison_byte >= 0)
337 memset(ptr, mem_poison_byte, pool->size);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200338 pool_put_to_cache(pool, ptr);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100339 }
340}
341
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100342#else /* CONFIG_HAP_LOCKLESS_POOLS */
Olivier Houchardcf975d42018-01-24 18:38:31 +0100343/*
Willy Tarreau02622412014-12-08 16:35:23 +0100344 * Returns a pointer to type <type> taken from the pool <pool_type> if
345 * available, otherwise returns NULL. No malloc() is attempted, and poisonning
346 * is never performed. The purpose is to get the fastest possible allocation.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200347 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200348static inline void *__pool_get_first(struct pool_head *pool)
Willy Tarreaue430e772014-12-23 14:13:16 +0100349{
350 void *p;
351
Willy Tarreau02622412014-12-08 16:35:23 +0100352 if ((p = pool->free_list) != NULL) {
Willy Tarreauac421112015-10-28 15:09:29 +0100353 pool->free_list = *POOL_LINK(pool, p);
Willy Tarreaue430e772014-12-23 14:13:16 +0100354 pool->used++;
Willy Tarreaude30a682015-10-28 15:23:51 +0100355#ifdef DEBUG_MEMORY_POOLS
356 /* keep track of where the element was allocated from */
357 *POOL_LINK(pool, p) = (void *)pool;
358#endif
Willy Tarreaue430e772014-12-23 14:13:16 +0100359 }
360 return p;
361}
Willy Tarreau50e608d2007-05-13 18:26:08 +0200362
Christopher Fauletb349e482017-08-29 09:52:38 +0200363static inline void *pool_get_first(struct pool_head *pool)
364{
365 void *ret;
366
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100367 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200368 ret = __pool_get_first(pool);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100369 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200370 return ret;
371}
Willy Tarreau50e608d2007-05-13 18:26:08 +0200372/*
Willy Tarreau02622412014-12-08 16:35:23 +0100373 * Returns a pointer to type <type> taken from the pool <pool_type> or
374 * dynamically allocated. In the first case, <pool_type> is updated to point to
375 * the next element in the list. No memory poisonning is ever performed on the
376 * returned area.
377 */
378static inline void *pool_alloc_dirty(struct pool_head *pool)
379{
380 void *p;
381
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100382 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200383 if ((p = __pool_get_first(pool)) == NULL)
384 p = __pool_refill_alloc(pool, 0);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100385 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreau02622412014-12-08 16:35:23 +0100386 return p;
387}
388
Willy Tarreau158fa752017-11-22 15:47:29 +0100389#ifndef DEBUG_UAF /* normal allocator */
390
Willy Tarreauf13322e2017-11-22 10:50:54 +0100391/* allocates an area of size <size> and returns it. The semantics are similar
392 * to those of malloc().
393 */
394static inline void *pool_alloc_area(size_t size)
395{
396 return malloc(size);
397}
398
399/* frees an area <area> of size <size> allocated by pool_alloc_area(). The
400 * semantics are identical to free() except that the size is specified and
401 * may be ignored.
402 */
403static inline void pool_free_area(void *area, size_t __maybe_unused size)
404{
405 free(area);
406}
407
Willy Tarreau158fa752017-11-22 15:47:29 +0100408#else /* use-after-free detector */
409
410/* allocates an area of size <size> and returns it. The semantics are similar
411 * to those of malloc(). However the allocation is rounded up to 4kB so that a
412 * full page is allocated. This ensures the object can be freed alone so that
413 * future dereferences are easily detected. The returned object is always
Willy Tarreau364d7452018-02-22 14:14:23 +0100414 * 16-bytes aligned to avoid issues with unaligned structure objects. In case
415 * some padding is added, the area's start address is copied at the end of the
416 * padding to help detect underflows.
Willy Tarreau158fa752017-11-22 15:47:29 +0100417 */
Olivier Houchard62975a72018-10-21 01:33:11 +0200418#include <errno.h>
Willy Tarreau158fa752017-11-22 15:47:29 +0100419static inline void *pool_alloc_area(size_t size)
420{
421 size_t pad = (4096 - size) & 0xFF0;
Willy Tarreau229e7392019-08-08 07:38:19 +0200422 int isolated;
Willy Tarreau5a9cce42018-02-22 11:39:23 +0100423 void *ret;
Willy Tarreau158fa752017-11-22 15:47:29 +0100424
Willy Tarreau229e7392019-08-08 07:38:19 +0200425 isolated = thread_isolated();
426 if (!isolated)
427 thread_harmless_now();
Olivier Houchard62975a72018-10-21 01:33:11 +0200428 ret = mmap(NULL, (size + 4095) & -4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
Willy Tarreau85b2cae2019-07-04 16:18:23 +0200429 if (ret != MAP_FAILED) {
430 /* let's dereference the page before returning so that the real
431 * allocation in the system is performed without holding the lock.
432 */
433 *(int *)ret = 0;
434 if (pad >= sizeof(void *))
435 *(void **)(ret + pad - sizeof(void *)) = ret + pad;
436 ret += pad;
437 } else {
438 ret = NULL;
439 }
Willy Tarreau229e7392019-08-08 07:38:19 +0200440 if (!isolated)
441 thread_harmless_end();
Willy Tarreau85b2cae2019-07-04 16:18:23 +0200442 return ret;
Willy Tarreau158fa752017-11-22 15:47:29 +0100443}
444
445/* frees an area <area> of size <size> allocated by pool_alloc_area(). The
446 * semantics are identical to free() except that the size must absolutely match
Willy Tarreau364d7452018-02-22 14:14:23 +0100447 * the one passed to pool_alloc_area(). In case some padding is added, the
448 * area's start address is compared to the one at the end of the padding, and
449 * a segfault is triggered if they don't match, indicating an underflow.
Willy Tarreau158fa752017-11-22 15:47:29 +0100450 */
451static inline void pool_free_area(void *area, size_t size)
452{
453 size_t pad = (4096 - size) & 0xFF0;
454
Willy Tarreau364d7452018-02-22 14:14:23 +0100455 if (pad >= sizeof(void *) && *(void **)(area - sizeof(void *)) != area)
Willy Tarreaue4d42552020-03-14 11:08:16 +0100456 *DISGUISE((volatile int *)0) = 0;
Willy Tarreau364d7452018-02-22 14:14:23 +0100457
Willy Tarreau85b2cae2019-07-04 16:18:23 +0200458 thread_harmless_now();
Willy Tarreau158fa752017-11-22 15:47:29 +0100459 munmap(area - pad, (size + 4095) & -4096);
Willy Tarreau85b2cae2019-07-04 16:18:23 +0200460 thread_harmless_end();
Willy Tarreau158fa752017-11-22 15:47:29 +0100461}
462
463#endif /* DEBUG_UAF */
464
Willy Tarreau02622412014-12-08 16:35:23 +0100465/*
466 * Returns a pointer to type <type> taken from the pool <pool_type> or
467 * dynamically allocated. In the first case, <pool_type> is updated to point to
468 * the next element in the list. Memory poisonning is performed if enabled.
469 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100470static inline void *pool_alloc(struct pool_head *pool)
Willy Tarreau02622412014-12-08 16:35:23 +0100471{
472 void *p;
473
474 p = pool_alloc_dirty(pool);
Willy Tarreaude30a682015-10-28 15:23:51 +0100475 if (p && mem_poison_byte >= 0) {
Willy Tarreau02622412014-12-08 16:35:23 +0100476 memset(p, mem_poison_byte, pool->size);
Willy Tarreaude30a682015-10-28 15:23:51 +0100477 }
478
Willy Tarreau02622412014-12-08 16:35:23 +0100479 return p;
480}
481
482/*
Willy Tarreau50e608d2007-05-13 18:26:08 +0200483 * Puts a memory area back to the corresponding pool.
484 * Items are chained directly through a pointer that
485 * is written in the beginning of the memory area, so
486 * there's no need for any carrier cell. This implies
487 * that each memory area is at least as big as one
Willy Tarreau48d63db2008-08-03 17:41:33 +0200488 * pointer. Just like with the libc's free(), nothing
489 * is done if <ptr> is NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200490 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100491static inline void pool_free(struct pool_head *pool, void *ptr)
Willy Tarreaue430e772014-12-23 14:13:16 +0100492{
493 if (likely(ptr != NULL)) {
Willy Tarreaude30a682015-10-28 15:23:51 +0100494#ifdef DEBUG_MEMORY_POOLS
495 /* we'll get late corruption if we refill to the wrong pool or double-free */
496 if (*POOL_LINK(pool, ptr) != (void *)pool)
Willy Tarreaue4d42552020-03-14 11:08:16 +0100497 *DISGUISE((volatile int *)0) = 0;
Willy Tarreaude30a682015-10-28 15:23:51 +0100498#endif
Willy Tarreau158fa752017-11-22 15:47:29 +0100499
500#ifndef DEBUG_UAF /* normal pool behaviour */
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200501 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreauac421112015-10-28 15:09:29 +0100502 *POOL_LINK(pool, ptr) = (void *)pool->free_list;
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200503 pool->free_list = (void *)ptr;
504 pool->used--;
505 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreau158fa752017-11-22 15:47:29 +0100506#else /* release the entry for real to detect use after free */
507 /* ensure we crash on double free or free of a const area*/
508 *(uint32_t *)ptr = 0xDEADADD4;
509 pool_free_area(ptr, pool->size + POOL_EXTRA);
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200510 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreau158fa752017-11-22 15:47:29 +0100511 pool->allocated--;
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200512 pool->used--;
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100513 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200514#endif /* DEBUG_UAF */
Willy Tarreaue430e772014-12-23 14:13:16 +0100515 }
516}
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100517#endif /* CONFIG_HAP_LOCKLESS_POOLS */
Willy Tarreau2dd0d472006-06-29 17:53:05 +0200518#endif /* _COMMON_MEMORY_H */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200519
520/*
521 * Local variables:
522 * c-indent-level: 8
523 * c-basic-offset: 8
524 * End:
525 */