Willy Tarreau | 3646777 | 2020-05-30 18:27:03 +0200 | [diff] [blame] | 1 | /* |
| 2 | * include/haproxy/pool-t.h |
| 3 | * Memory pools configuration and type definitions. |
| 4 | * |
| 5 | * Copyright (C) 2000-2020 Willy Tarreau - w@1wt.eu |
| 6 | * |
| 7 | * This library is free software; you can redistribute it and/or |
| 8 | * modify it under the terms of the GNU Lesser General Public |
| 9 | * License as published by the Free Software Foundation, version 2.1 |
| 10 | * exclusively. |
| 11 | * |
| 12 | * This library is distributed in the hope that it will be useful, |
| 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 15 | * Lesser General Public License for more details. |
| 16 | * |
| 17 | * You should have received a copy of the GNU Lesser General Public |
| 18 | * License along with this library; if not, write to the Free Software |
| 19 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| 20 | */ |
| 21 | |
| 22 | #ifndef _HAPROXY_POOL_T_H |
| 23 | #define _HAPROXY_POOL_T_H |
| 24 | |
Willy Tarreau | c03d763 | 2020-06-29 10:11:24 +0200 | [diff] [blame] | 25 | #include <haproxy/api-t.h> |
| 26 | #include <haproxy/list-t.h> |
| 27 | #include <haproxy/thread-t.h> |
Willy Tarreau | 3646777 | 2020-05-30 18:27:03 +0200 | [diff] [blame] | 28 | |
Willy Tarreau | 2d6f628 | 2021-04-15 16:24:00 +0200 | [diff] [blame] | 29 | /* Pools are always enabled unless explicitly disabled. When disabled, the |
| 30 | * calls are directly passed to the underlying OS functions. |
| 31 | */ |
| 32 | #if !defined(DEBUG_NO_POOLS) && !defined(DEBUG_UAF) && !defined(DEBUG_FAIL_ALLOC) |
| 33 | #define CONFIG_HAP_POOLS |
| 34 | #endif |
| 35 | |
Willy Tarreau | 3646777 | 2020-05-30 18:27:03 +0200 | [diff] [blame] | 36 | /* On architectures supporting threads and double-word CAS, we can implement |
| 37 | * lock-less memory pools. This isn't supported for debugging modes however. |
| 38 | */ |
Willy Tarreau | b8498e9 | 2021-04-18 10:23:02 +0200 | [diff] [blame] | 39 | #if defined(USE_THREAD) && defined(HA_HAVE_CAS_DW) && defined(CONFIG_HAP_POOLS) && !defined(DEBUG_NO_LOCKLESS_POOLS) |
Willy Tarreau | 3646777 | 2020-05-30 18:27:03 +0200 | [diff] [blame] | 40 | #define CONFIG_HAP_LOCKLESS_POOLS |
| 41 | #endif |
| 42 | |
Willy Tarreau | 0bae075 | 2021-03-02 20:05:09 +0100 | [diff] [blame] | 43 | /* On modern architectures with many threads, a fast memory allocator, and |
| 44 | * local pools, the global pools with their single list can be way slower than |
| 45 | * the standard allocator which already has its own per-thread arenas. In this |
| 46 | * case we disable global pools. The global pools may still be enforced |
| 47 | * using CONFIG_HAP_GLOBAL_POOLS though. |
| 48 | */ |
Willy Tarreau | 53a7fe4 | 2021-04-15 16:43:18 +0200 | [diff] [blame] | 49 | #if defined(USE_THREAD) && defined(HA_HAVE_FAST_MALLOC) && !defined(CONFIG_HAP_GLOBAL_POOLS) |
Willy Tarreau | 0bae075 | 2021-03-02 20:05:09 +0100 | [diff] [blame] | 50 | #define CONFIG_HAP_NO_GLOBAL_POOLS |
| 51 | #endif |
| 52 | |
Willy Tarreau | 3646777 | 2020-05-30 18:27:03 +0200 | [diff] [blame] | 53 | #define MEM_F_SHARED 0x1 |
Willy Tarreau | 3646777 | 2020-05-30 18:27:03 +0200 | [diff] [blame] | 54 | #define MEM_F_EXACT 0x2 |
| 55 | |
| 56 | /* By default, free objects are linked by a pointer stored at the beginning of |
| 57 | * the memory area. When DEBUG_MEMORY_POOLS is set, the allocated area is |
| 58 | * inflated by the size of a pointer so that the link is placed at the end |
| 59 | * of the objects. Hence free objects in pools remain intact. In addition, |
| 60 | * this location is used to keep a pointer to the pool the object was |
| 61 | * allocated from, and verify it's freed into the appropriate one. |
| 62 | */ |
| 63 | #ifdef DEBUG_MEMORY_POOLS |
| 64 | #define POOL_EXTRA (sizeof(void *)) |
| 65 | #define POOL_LINK(pool, item) (void **)(((char *)(item)) + ((pool)->size)) |
| 66 | #else |
| 67 | #define POOL_EXTRA (0) |
| 68 | #define POOL_LINK(pool, item) ((void **)(item)) |
| 69 | #endif |
| 70 | |
Willy Tarreau | 3646777 | 2020-05-30 18:27:03 +0200 | [diff] [blame] | 71 | #define POOL_AVG_SAMPLES 1024 |
| 72 | |
Willy Tarreau | de749a9 | 2021-03-22 20:54:15 +0100 | [diff] [blame] | 73 | /* possible flags for __pool_alloc() */ |
| 74 | #define POOL_F_NO_POISON 0x00000001 // do not poison the area |
| 75 | #define POOL_F_MUST_ZERO 0x00000002 // zero the returned area |
Willy Tarreau | 207c095 | 2021-04-17 16:00:08 +0200 | [diff] [blame] | 76 | #define POOL_F_NO_FAIL 0x00000004 // do not randomly fail |
Willy Tarreau | de749a9 | 2021-03-22 20:54:15 +0100 | [diff] [blame] | 77 | |
Willy Tarreau | 3646777 | 2020-05-30 18:27:03 +0200 | [diff] [blame] | 78 | |
| 79 | struct pool_cache_head { |
| 80 | struct list list; /* head of objects in this pool */ |
Willy Tarreau | 3646777 | 2020-05-30 18:27:03 +0200 | [diff] [blame] | 81 | unsigned int count; /* number of objects in this pool */ |
Willy Tarreau | 9f3129e | 2021-04-17 00:31:38 +0200 | [diff] [blame] | 82 | } THREAD_ALIGNED(64); |
Willy Tarreau | 3646777 | 2020-05-30 18:27:03 +0200 | [diff] [blame] | 83 | |
| 84 | struct pool_cache_item { |
| 85 | struct list by_pool; /* link to objects in this pool */ |
| 86 | struct list by_lru; /* link to objects by LRU order */ |
| 87 | }; |
| 88 | |
| 89 | struct pool_free_list { |
| 90 | void **free_list; |
| 91 | uintptr_t seq; |
| 92 | }; |
| 93 | |
| 94 | /* Note below, in case of lockless pools, we still need the lock only for |
| 95 | * the flush() operation. |
| 96 | */ |
| 97 | struct pool_head { |
| 98 | void **free_list; |
| 99 | #ifdef CONFIG_HAP_LOCKLESS_POOLS |
| 100 | uintptr_t seq; |
| 101 | #endif |
| 102 | __decl_thread(HA_SPINLOCK_T lock); /* the spin lock */ |
| 103 | unsigned int used; /* how many chunks are currently in use */ |
| 104 | unsigned int needed_avg;/* floating indicator between used and allocated */ |
| 105 | unsigned int allocated; /* how many chunks have been allocated */ |
| 106 | unsigned int limit; /* hard limit on the number of chunks */ |
| 107 | unsigned int minavail; /* how many chunks are expected to be used */ |
| 108 | unsigned int size; /* chunk size */ |
| 109 | unsigned int flags; /* MEM_F_* */ |
| 110 | unsigned int users; /* number of pools sharing this zone */ |
| 111 | unsigned int failed; /* failed allocations */ |
| 112 | struct list list; /* list of all known pools */ |
| 113 | char name[12]; /* name of the pool */ |
Willy Tarreau | 2d6f628 | 2021-04-15 16:24:00 +0200 | [diff] [blame] | 114 | #ifdef CONFIG_HAP_POOLS |
Willy Tarreau | 9f3129e | 2021-04-17 00:31:38 +0200 | [diff] [blame] | 115 | struct pool_cache_head cache[MAX_THREADS]; /* pool caches */ |
| 116 | #endif |
Willy Tarreau | 3646777 | 2020-05-30 18:27:03 +0200 | [diff] [blame] | 117 | } __attribute__((aligned(64))); |
| 118 | |
| 119 | #endif /* _HAPROXY_POOL_T_H */ |
| 120 | |
| 121 | /* |
| 122 | * Local variables: |
| 123 | * c-indent-level: 8 |
| 124 | * c-basic-offset: 8 |
| 125 | * End: |
| 126 | */ |