blob: 507198966b2f0b293ace077c7061f9259aab9384 [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
Willy Tarreau62405a22014-12-23 13:51:28 +01002 * include/common/memory.h
3 * Memory management definitions..
4 *
5 * Copyright (C) 2000-2014 Willy Tarreau - w@1wt.eu
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation, version 2.1
10 * exclusively.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
Willy Tarreaubaaee002006-06-26 02:48:02 +020021
Willy Tarreau2dd0d472006-06-29 17:53:05 +020022#ifndef _COMMON_MEMORY_H
23#define _COMMON_MEMORY_H
Willy Tarreaubaaee002006-06-26 02:48:02 +020024
Willy Tarreau158fa752017-11-22 15:47:29 +010025#include <sys/mman.h>
26
Willy Tarreaubaaee002006-06-26 02:48:02 +020027#include <stdlib.h>
Willy Tarreaue430e772014-12-23 14:13:16 +010028#include <string.h>
Willy Tarreaua1bd1fa2019-03-29 17:26:33 +010029#include <inttypes.h>
Willy Tarreaua7280a12018-11-26 19:41:40 +010030#include <unistd.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020031
Willy Tarreau2dd0d472006-06-29 17:53:05 +020032#include <common/config.h>
Willy Tarreau50e608d2007-05-13 18:26:08 +020033#include <common/mini-clist.h>
Christopher Fauletb349e482017-08-29 09:52:38 +020034#include <common/hathreads.h>
Willy Tarreau7107c8b2018-11-26 11:44:35 +010035#include <common/initcall.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020036
Willy Tarreaua84dcb82015-10-28 12:04:02 +010037#ifndef DEBUG_DONT_SHARE_POOLS
Willy Tarreau50e608d2007-05-13 18:26:08 +020038#define MEM_F_SHARED 0x1
Willy Tarreaua84dcb82015-10-28 12:04:02 +010039#else
40#define MEM_F_SHARED 0
41#endif
Willy Tarreau581bf812016-01-25 02:19:13 +010042#define MEM_F_EXACT 0x2
Willy Tarreau50e608d2007-05-13 18:26:08 +020043
Willy Tarreauac421112015-10-28 15:09:29 +010044/* reserve an extra void* at the end of a pool for linking */
45#ifdef DEBUG_MEMORY_POOLS
46#define POOL_EXTRA (sizeof(void *))
47#define POOL_LINK(pool, item) (void **)(((char *)item) + (pool->size))
48#else
49#define POOL_EXTRA (0)
50#define POOL_LINK(pool, item) ((void **)(item))
51#endif
52
Willy Tarreau0a93b642018-10-16 07:58:39 +020053#define MAX_BASE_POOLS 32
54
Willy Tarreaue18db9e2018-10-16 10:28:54 +020055struct pool_cache_head {
56 struct list list; /* head of objects in this pool */
57 size_t size; /* size of an object */
58 unsigned int count; /* number of objects in this pool */
59};
60
61struct pool_cache_item {
62 struct list by_pool; /* link to objects in this pool */
63 struct list by_lru; /* link to objects by LRU order */
64};
65
Willy Tarreau7f0165e2018-11-26 17:09:46 +010066extern struct pool_cache_head pool_cache[][MAX_BASE_POOLS];
Willy Tarreaue18db9e2018-10-16 10:28:54 +020067extern THREAD_LOCAL size_t pool_cache_bytes; /* total cache size */
68extern THREAD_LOCAL size_t pool_cache_count; /* #cache objects */
69
Willy Tarreauf161d0f2018-02-22 14:05:55 +010070#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +010071struct pool_free_list {
72 void **free_list;
73 uintptr_t seq;
74};
75#endif
76
Willy Tarreau50e608d2007-05-13 18:26:08 +020077struct pool_head {
Willy Tarreau1ca1b702017-11-26 10:50:36 +010078 void **free_list;
Willy Tarreauf161d0f2018-02-22 14:05:55 +010079#ifdef CONFIG_HAP_LOCKLESS_POOLS
Olivier Houchardcf975d42018-01-24 18:38:31 +010080 uintptr_t seq;
Olivier Houchard04f5fe82020-02-01 17:49:31 +010081 HA_RWLOCK_T flush_lock;
Olivier Houchardcf975d42018-01-24 18:38:31 +010082#else
83 __decl_hathreads(HA_SPINLOCK_T lock); /* the spin lock */
84#endif
Willy Tarreau50e608d2007-05-13 18:26:08 +020085 unsigned int used; /* how many chunks are currently in use */
86 unsigned int allocated; /* how many chunks have been allocated */
87 unsigned int limit; /* hard limit on the number of chunks */
88 unsigned int minavail; /* how many chunks are expected to be used */
89 unsigned int size; /* chunk size */
90 unsigned int flags; /* MEM_F_* */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020091 unsigned int users; /* number of pools sharing this zone */
Willy Tarreau58102cf2015-10-28 16:24:21 +010092 unsigned int failed; /* failed allocations */
Olivier Houchardcf975d42018-01-24 18:38:31 +010093 struct list list; /* list of all known pools */
Willy Tarreau7dcd46d2007-05-14 00:16:13 +020094 char name[12]; /* name of the pool */
Willy Tarreau1ca1b702017-11-26 10:50:36 +010095} __attribute__((aligned(64)));
Willy Tarreau50e608d2007-05-13 18:26:08 +020096
Willy Tarreau0a93b642018-10-16 07:58:39 +020097
98extern struct pool_head pool_base_start[MAX_BASE_POOLS];
99extern unsigned int pool_base_count;
100
Willy Tarreau067ac9f2015-10-08 14:12:13 +0200101/* poison each newly allocated area with this byte if >= 0 */
102extern int mem_poison_byte;
Willy Tarreau50e608d2007-05-13 18:26:08 +0200103
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100104/* Allocates new entries for pool <pool> until there are at least <avail> + 1
105 * available, then returns the last one for immediate use, so that at least
106 * <avail> are left available in the pool upon return. NULL is returned if the
107 * last entry could not be allocated. It's important to note that at least one
108 * allocation is always performed even if there are enough entries in the pool.
109 * A call to the garbage collector is performed at most once in case malloc()
110 * returns an error, before returning NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200111 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200112void *__pool_refill_alloc(struct pool_head *pool, unsigned int avail);
Willy Tarreaua885f6d2014-12-03 15:25:28 +0100113void *pool_refill_alloc(struct pool_head *pool, unsigned int avail);
Willy Tarreau50e608d2007-05-13 18:26:08 +0200114
115/* Try to find an existing shared pool with the same characteristics and
116 * returns it, otherwise creates this one. NULL is returned if no memory
117 * is available for a new creation.
118 */
119struct pool_head *create_pool(char *name, unsigned int size, unsigned int flags);
Willy Tarreau7107c8b2018-11-26 11:44:35 +0100120void create_pool_callback(struct pool_head **ptr, char *name, unsigned int size);
121
122/* This registers a call to create_pool_callback(ptr, name, size) */
123#define REGISTER_POOL(ptr, name, size) \
124 INITCALL3(STG_POOL, create_pool_callback, (ptr), (name), (size))
125
126/* This macro declares a pool head <ptr> and registers its creation */
127#define DECLARE_POOL(ptr, name, size) \
128 struct pool_head *(ptr) = NULL; \
129 REGISTER_POOL(&ptr, name, size)
130
131/* This macro declares a static pool head <ptr> and registers its creation */
132#define DECLARE_STATIC_POOL(ptr, name, size) \
133 static struct pool_head *(ptr); \
134 REGISTER_POOL(&ptr, name, size)
Willy Tarreau50e608d2007-05-13 18:26:08 +0200135
136/* Dump statistics on pools usage.
137 */
Willy Tarreau12833bb2014-01-28 16:49:56 +0100138void dump_pools_to_trash();
Willy Tarreau50e608d2007-05-13 18:26:08 +0200139void dump_pools(void);
Willy Tarreau58102cf2015-10-28 16:24:21 +0100140int pool_total_failures();
141unsigned long pool_total_allocated();
142unsigned long pool_total_used();
Willy Tarreau50e608d2007-05-13 18:26:08 +0200143
144/*
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200145 * This function frees whatever can be freed in pool <pool>.
146 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100147void pool_flush(struct pool_head *pool);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200148
149/*
150 * This function frees whatever can be freed in all pools, but respecting
151 * the minimum thresholds imposed by owners.
Christopher Fauletb349e482017-08-29 09:52:38 +0200152 *
Willy Tarreaubafbe012017-11-24 17:34:44 +0100153 * <pool_ctx> is used when pool_gc is called to release resources to allocate
Christopher Fauletb349e482017-08-29 09:52:38 +0200154 * an element in __pool_refill_alloc. It is important because <pool_ctx> is
155 * already locked, so we need to skip the lock here.
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200156 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100157void pool_gc(struct pool_head *pool_ctx);
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200158
159/*
160 * This function destroys a pull by freeing it completely.
161 * This should be called only under extreme circumstances.
162 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100163void *pool_destroy(struct pool_head *pool);
Willy Tarreau2455ceb2018-11-26 15:57:34 +0100164void pool_destroy_all();
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200165
Willy Tarreau0a93b642018-10-16 07:58:39 +0200166/* returns the pool index for pool <pool>, or -1 if this pool has no index */
167static inline ssize_t pool_get_index(const struct pool_head *pool)
168{
169 size_t idx;
170
171 idx = pool - pool_base_start;
172 if (idx >= MAX_BASE_POOLS)
173 return -1;
174 return idx;
175}
176
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100177#ifdef CONFIG_HAP_LOCKLESS_POOLS
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200178
179/* Tries to retrieve an object from the local pool cache corresponding to pool
180 * <pool>. Returns NULL if none is available.
181 */
182static inline void *__pool_get_from_cache(struct pool_head *pool)
183{
184 ssize_t idx = pool_get_index(pool);
185 struct pool_cache_item *item;
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100186 struct pool_cache_head *ph;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200187
188 /* pool not in cache */
189 if (idx < 0)
190 return NULL;
191
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100192 ph = &pool_cache[tid][idx];
193 if (LIST_ISEMPTY(&ph->list))
194 return NULL; // empty
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200195
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100196 item = LIST_NEXT(&ph->list, typeof(item), by_pool);
197 ph->count--;
198 pool_cache_bytes -= ph->size;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200199 pool_cache_count--;
200 LIST_DEL(&item->by_pool);
201 LIST_DEL(&item->by_lru);
Willy Tarreau8e9f4532018-10-28 20:09:12 +0100202#ifdef DEBUG_MEMORY_POOLS
203 /* keep track of where the element was allocated from */
204 *POOL_LINK(pool, item) = (void *)pool;
205#endif
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200206 return item;
207}
208
Olivier Houchardcf975d42018-01-24 18:38:31 +0100209/*
210 * Returns a pointer to type <type> taken from the pool <pool_type> if
211 * available, otherwise returns NULL. No malloc() is attempted, and poisonning
212 * is never performed. The purpose is to get the fastest possible allocation.
213 */
214static inline void *__pool_get_first(struct pool_head *pool)
215{
216 struct pool_free_list cmp, new;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200217 void *ret = __pool_get_from_cache(pool);
218
219 if (ret)
220 return ret;
Olivier Houchardcf975d42018-01-24 18:38:31 +0100221
222 cmp.seq = pool->seq;
223 __ha_barrier_load();
224
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100225 HA_RWLOCK_RDLOCK(POOL_LOCK, &pool->flush_lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100226 cmp.free_list = pool->free_list;
227 do {
Olivier Houchard1c7c0d6b2020-02-03 13:03:30 +0100228 if (cmp.free_list == NULL) {
229 HA_RWLOCK_RDUNLOCK(POOL_LOCK, &pool->flush_lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100230 return NULL;
Olivier Houchard1c7c0d6b2020-02-03 13:03:30 +0100231 }
Olivier Houchardcf975d42018-01-24 18:38:31 +0100232 new.seq = cmp.seq + 1;
233 __ha_barrier_load();
234 new.free_list = *POOL_LINK(pool, cmp.free_list);
Willy Tarreau6a38b322019-05-11 18:04:24 +0200235 } while (HA_ATOMIC_DWCAS((void *)&pool->free_list, (void *)&cmp, (void *)&new) == 0);
Olivier Houchard20872762019-03-08 18:53:35 +0100236 __ha_barrier_atomic_store();
Olivier Houchard04f5fe82020-02-01 17:49:31 +0100237 HA_RWLOCK_RDUNLOCK(POOL_LOCK, &pool->flush_lock);
Tim Duesterhus05f6a432018-02-20 00:49:46 +0100238
Olivier Houchard20872762019-03-08 18:53:35 +0100239 _HA_ATOMIC_ADD(&pool->used, 1);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100240#ifdef DEBUG_MEMORY_POOLS
241 /* keep track of where the element was allocated from */
242 *POOL_LINK(pool, cmp.free_list) = (void *)pool;
243#endif
244 return cmp.free_list;
245}
246
247static inline void *pool_get_first(struct pool_head *pool)
248{
249 void *ret;
250
251 ret = __pool_get_first(pool);
252 return ret;
253}
254/*
255 * Returns a pointer to type <type> taken from the pool <pool_type> or
256 * dynamically allocated. In the first case, <pool_type> is updated to point to
257 * the next element in the list. No memory poisonning is ever performed on the
258 * returned area.
259 */
260static inline void *pool_alloc_dirty(struct pool_head *pool)
261{
262 void *p;
263
264 if ((p = __pool_get_first(pool)) == NULL)
265 p = __pool_refill_alloc(pool, 0);
266 return p;
267}
268
Willy Tarreaue6ce59d2007-05-13 19:38:49 +0200269/*
Olivier Houchardcf975d42018-01-24 18:38:31 +0100270 * Returns a pointer to type <type> taken from the pool <pool_type> or
271 * dynamically allocated. In the first case, <pool_type> is updated to point to
272 * the next element in the list. Memory poisonning is performed if enabled.
273 */
274static inline void *pool_alloc(struct pool_head *pool)
275{
276 void *p;
277
278 p = pool_alloc_dirty(pool);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100279 if (p && mem_poison_byte >= 0) {
280 memset(p, mem_poison_byte, pool->size);
281 }
282
283 return p;
284}
285
Willy Tarreau146794d2018-10-16 08:55:15 +0200286/* Locklessly add item <ptr> to pool <pool>, then update the pool used count.
287 * Both the pool and the pointer must be valid. Use pool_free() for normal
288 * operations.
289 */
290static inline void __pool_free(struct pool_head *pool, void *ptr)
291{
Willy Tarreau7a6ad882018-10-20 17:37:38 +0200292 void **free_list = pool->free_list;
Willy Tarreau146794d2018-10-16 08:55:15 +0200293
294 do {
295 *POOL_LINK(pool, ptr) = (void *)free_list;
296 __ha_barrier_store();
Olivier Houchard20872762019-03-08 18:53:35 +0100297 } while (!_HA_ATOMIC_CAS(&pool->free_list, &free_list, ptr));
298 __ha_barrier_atomic_store();
299 _HA_ATOMIC_SUB(&pool->used, 1);
Willy Tarreau146794d2018-10-16 08:55:15 +0200300}
301
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200302/* frees an object to the local cache, possibly pushing oldest objects to the
303 * global pool.
304 */
305void __pool_put_to_cache(struct pool_head *pool, void *ptr, ssize_t idx);
306static inline void pool_put_to_cache(struct pool_head *pool, void *ptr)
307{
308 ssize_t idx = pool_get_index(pool);
309
310 /* pool not in cache or too many objects for this pool (more than
311 * half of the cache is used and this pool uses more than 1/8 of
312 * the cache size).
313 */
314 if (idx < 0 ||
315 (pool_cache_bytes > CONFIG_HAP_POOL_CACHE_SIZE * 3 / 4 &&
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100316 pool_cache[tid][idx].count >= 16 + pool_cache_count / 8)) {
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200317 __pool_free(pool, ptr);
318 return;
319 }
320 __pool_put_to_cache(pool, ptr, idx);
321}
322
Olivier Houchardcf975d42018-01-24 18:38:31 +0100323/*
324 * Puts a memory area back to the corresponding pool.
325 * Items are chained directly through a pointer that
326 * is written in the beginning of the memory area, so
327 * there's no need for any carrier cell. This implies
328 * that each memory area is at least as big as one
329 * pointer. Just like with the libc's free(), nothing
330 * is done if <ptr> is NULL.
331 */
332static inline void pool_free(struct pool_head *pool, void *ptr)
333{
334 if (likely(ptr != NULL)) {
Olivier Houchardcf975d42018-01-24 18:38:31 +0100335#ifdef DEBUG_MEMORY_POOLS
336 /* we'll get late corruption if we refill to the wrong pool or double-free */
337 if (*POOL_LINK(pool, ptr) != (void *)pool)
338 *(volatile int *)0 = 0;
339#endif
Willy Tarreauda520352019-11-15 06:59:54 +0100340 if (mem_poison_byte >= 0)
341 memset(ptr, mem_poison_byte, pool->size);
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200342 pool_put_to_cache(pool, ptr);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100343 }
344}
345
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100346#else /* CONFIG_HAP_LOCKLESS_POOLS */
Olivier Houchardcf975d42018-01-24 18:38:31 +0100347/*
Willy Tarreau02622412014-12-08 16:35:23 +0100348 * Returns a pointer to type <type> taken from the pool <pool_type> if
349 * available, otherwise returns NULL. No malloc() is attempted, and poisonning
350 * is never performed. The purpose is to get the fastest possible allocation.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200351 */
Christopher Fauletb349e482017-08-29 09:52:38 +0200352static inline void *__pool_get_first(struct pool_head *pool)
Willy Tarreaue430e772014-12-23 14:13:16 +0100353{
354 void *p;
355
Willy Tarreau02622412014-12-08 16:35:23 +0100356 if ((p = pool->free_list) != NULL) {
Willy Tarreauac421112015-10-28 15:09:29 +0100357 pool->free_list = *POOL_LINK(pool, p);
Willy Tarreaue430e772014-12-23 14:13:16 +0100358 pool->used++;
Willy Tarreaude30a682015-10-28 15:23:51 +0100359#ifdef DEBUG_MEMORY_POOLS
360 /* keep track of where the element was allocated from */
361 *POOL_LINK(pool, p) = (void *)pool;
362#endif
Willy Tarreaue430e772014-12-23 14:13:16 +0100363 }
364 return p;
365}
Willy Tarreau50e608d2007-05-13 18:26:08 +0200366
Christopher Fauletb349e482017-08-29 09:52:38 +0200367static inline void *pool_get_first(struct pool_head *pool)
368{
369 void *ret;
370
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100371 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200372 ret = __pool_get_first(pool);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100373 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200374 return ret;
375}
Willy Tarreau50e608d2007-05-13 18:26:08 +0200376/*
Willy Tarreau02622412014-12-08 16:35:23 +0100377 * Returns a pointer to type <type> taken from the pool <pool_type> or
378 * dynamically allocated. In the first case, <pool_type> is updated to point to
379 * the next element in the list. No memory poisonning is ever performed on the
380 * returned area.
381 */
382static inline void *pool_alloc_dirty(struct pool_head *pool)
383{
384 void *p;
385
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100386 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Christopher Fauletb349e482017-08-29 09:52:38 +0200387 if ((p = __pool_get_first(pool)) == NULL)
388 p = __pool_refill_alloc(pool, 0);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100389 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreau02622412014-12-08 16:35:23 +0100390 return p;
391}
392
Willy Tarreau158fa752017-11-22 15:47:29 +0100393#ifndef DEBUG_UAF /* normal allocator */
394
Willy Tarreauf13322e2017-11-22 10:50:54 +0100395/* allocates an area of size <size> and returns it. The semantics are similar
396 * to those of malloc().
397 */
398static inline void *pool_alloc_area(size_t size)
399{
400 return malloc(size);
401}
402
403/* frees an area <area> of size <size> allocated by pool_alloc_area(). The
404 * semantics are identical to free() except that the size is specified and
405 * may be ignored.
406 */
407static inline void pool_free_area(void *area, size_t __maybe_unused size)
408{
409 free(area);
410}
411
Willy Tarreau158fa752017-11-22 15:47:29 +0100412#else /* use-after-free detector */
413
414/* allocates an area of size <size> and returns it. The semantics are similar
415 * to those of malloc(). However the allocation is rounded up to 4kB so that a
416 * full page is allocated. This ensures the object can be freed alone so that
417 * future dereferences are easily detected. The returned object is always
Willy Tarreau364d7452018-02-22 14:14:23 +0100418 * 16-bytes aligned to avoid issues with unaligned structure objects. In case
419 * some padding is added, the area's start address is copied at the end of the
420 * padding to help detect underflows.
Willy Tarreau158fa752017-11-22 15:47:29 +0100421 */
Olivier Houchard62975a72018-10-21 01:33:11 +0200422#include <errno.h>
Willy Tarreau158fa752017-11-22 15:47:29 +0100423static inline void *pool_alloc_area(size_t size)
424{
425 size_t pad = (4096 - size) & 0xFF0;
Willy Tarreau229e7392019-08-08 07:38:19 +0200426 int isolated;
Willy Tarreau5a9cce42018-02-22 11:39:23 +0100427 void *ret;
Willy Tarreau158fa752017-11-22 15:47:29 +0100428
Willy Tarreau229e7392019-08-08 07:38:19 +0200429 isolated = thread_isolated();
430 if (!isolated)
431 thread_harmless_now();
Olivier Houchard62975a72018-10-21 01:33:11 +0200432 ret = mmap(NULL, (size + 4095) & -4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
Willy Tarreau85b2cae2019-07-04 16:18:23 +0200433 if (ret != MAP_FAILED) {
434 /* let's dereference the page before returning so that the real
435 * allocation in the system is performed without holding the lock.
436 */
437 *(int *)ret = 0;
438 if (pad >= sizeof(void *))
439 *(void **)(ret + pad - sizeof(void *)) = ret + pad;
440 ret += pad;
441 } else {
442 ret = NULL;
443 }
Willy Tarreau229e7392019-08-08 07:38:19 +0200444 if (!isolated)
445 thread_harmless_end();
Willy Tarreau85b2cae2019-07-04 16:18:23 +0200446 return ret;
Willy Tarreau158fa752017-11-22 15:47:29 +0100447}
448
449/* frees an area <area> of size <size> allocated by pool_alloc_area(). The
450 * semantics are identical to free() except that the size must absolutely match
Willy Tarreau364d7452018-02-22 14:14:23 +0100451 * the one passed to pool_alloc_area(). In case some padding is added, the
452 * area's start address is compared to the one at the end of the padding, and
453 * a segfault is triggered if they don't match, indicating an underflow.
Willy Tarreau158fa752017-11-22 15:47:29 +0100454 */
455static inline void pool_free_area(void *area, size_t size)
456{
457 size_t pad = (4096 - size) & 0xFF0;
458
Willy Tarreau364d7452018-02-22 14:14:23 +0100459 if (pad >= sizeof(void *) && *(void **)(area - sizeof(void *)) != area)
460 *(volatile int *)0 = 0;
461
Willy Tarreau85b2cae2019-07-04 16:18:23 +0200462 thread_harmless_now();
Willy Tarreau158fa752017-11-22 15:47:29 +0100463 munmap(area - pad, (size + 4095) & -4096);
Willy Tarreau85b2cae2019-07-04 16:18:23 +0200464 thread_harmless_end();
Willy Tarreau158fa752017-11-22 15:47:29 +0100465}
466
467#endif /* DEBUG_UAF */
468
Willy Tarreau02622412014-12-08 16:35:23 +0100469/*
470 * Returns a pointer to type <type> taken from the pool <pool_type> or
471 * dynamically allocated. In the first case, <pool_type> is updated to point to
472 * the next element in the list. Memory poisonning is performed if enabled.
473 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100474static inline void *pool_alloc(struct pool_head *pool)
Willy Tarreau02622412014-12-08 16:35:23 +0100475{
476 void *p;
477
478 p = pool_alloc_dirty(pool);
Willy Tarreaude30a682015-10-28 15:23:51 +0100479 if (p && mem_poison_byte >= 0) {
Willy Tarreau02622412014-12-08 16:35:23 +0100480 memset(p, mem_poison_byte, pool->size);
Willy Tarreaude30a682015-10-28 15:23:51 +0100481 }
482
Willy Tarreau02622412014-12-08 16:35:23 +0100483 return p;
484}
485
486/*
Willy Tarreau50e608d2007-05-13 18:26:08 +0200487 * Puts a memory area back to the corresponding pool.
488 * Items are chained directly through a pointer that
489 * is written in the beginning of the memory area, so
490 * there's no need for any carrier cell. This implies
491 * that each memory area is at least as big as one
Willy Tarreau48d63db2008-08-03 17:41:33 +0200492 * pointer. Just like with the libc's free(), nothing
493 * is done if <ptr> is NULL.
Willy Tarreau50e608d2007-05-13 18:26:08 +0200494 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100495static inline void pool_free(struct pool_head *pool, void *ptr)
Willy Tarreaue430e772014-12-23 14:13:16 +0100496{
497 if (likely(ptr != NULL)) {
Willy Tarreaude30a682015-10-28 15:23:51 +0100498#ifdef DEBUG_MEMORY_POOLS
499 /* we'll get late corruption if we refill to the wrong pool or double-free */
500 if (*POOL_LINK(pool, ptr) != (void *)pool)
Olivier Houchard51e47412018-12-16 00:58:28 +0100501 *(volatile int *)0 = 0;
Willy Tarreaude30a682015-10-28 15:23:51 +0100502#endif
Willy Tarreau158fa752017-11-22 15:47:29 +0100503
504#ifndef DEBUG_UAF /* normal pool behaviour */
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200505 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreauac421112015-10-28 15:09:29 +0100506 *POOL_LINK(pool, ptr) = (void *)pool->free_list;
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200507 pool->free_list = (void *)ptr;
508 pool->used--;
509 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreau158fa752017-11-22 15:47:29 +0100510#else /* release the entry for real to detect use after free */
511 /* ensure we crash on double free or free of a const area*/
512 *(uint32_t *)ptr = 0xDEADADD4;
513 pool_free_area(ptr, pool->size + POOL_EXTRA);
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200514 HA_SPIN_LOCK(POOL_LOCK, &pool->lock);
Willy Tarreau158fa752017-11-22 15:47:29 +0100515 pool->allocated--;
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200516 pool->used--;
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100517 HA_SPIN_UNLOCK(POOL_LOCK, &pool->lock);
Willy Tarreau3e853ea2019-07-04 11:30:00 +0200518#endif /* DEBUG_UAF */
Willy Tarreaue430e772014-12-23 14:13:16 +0100519 }
520}
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100521#endif /* CONFIG_HAP_LOCKLESS_POOLS */
Willy Tarreau2dd0d472006-06-29 17:53:05 +0200522#endif /* _COMMON_MEMORY_H */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200523
524/*
525 * Local variables:
526 * c-indent-level: 8
527 * c-basic-offset: 8
528 * End:
529 */