Emeric Brun | 3e541d1 | 2012-09-03 11:14:36 +0200 | [diff] [blame] | 1 | /* |
Willy Tarreau | 334099c | 2020-06-03 18:38:48 +0200 | [diff] [blame] | 2 | * include/haproxy/shctx.h - shared context management functions for SSL |
Emeric Brun | 3e541d1 | 2012-09-03 11:14:36 +0200 | [diff] [blame] | 3 | * |
| 4 | * Copyright (C) 2011-2012 EXCELIANCE |
| 5 | * |
| 6 | * Author: Emeric Brun - emeric@exceliance.fr |
| 7 | * |
| 8 | * This program is free software; you can redistribute it and/or |
| 9 | * modify it under the terms of the GNU General Public License |
| 10 | * as published by the Free Software Foundation; either version |
| 11 | * 2 of the License, or (at your option) any later version. |
| 12 | */ |
| 13 | |
Willy Tarreau | 334099c | 2020-06-03 18:38:48 +0200 | [diff] [blame] | 14 | #ifndef __HAPROXY_SHCTX_H |
| 15 | #define __HAPROXY_SHCTX_H |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 16 | |
Willy Tarreau | ca36771 | 2021-06-15 15:03:19 +0200 | [diff] [blame] | 17 | #include <haproxy/api.h> |
Willy Tarreau | 853b297 | 2020-05-27 18:01:47 +0200 | [diff] [blame] | 18 | #include <haproxy/list.h> |
Willy Tarreau | 334099c | 2020-06-03 18:38:48 +0200 | [diff] [blame] | 19 | #include <haproxy/shctx-t.h> |
Emeric Brun | 3e541d1 | 2012-09-03 11:14:36 +0200 | [diff] [blame] | 20 | |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 21 | #ifndef USE_PRIVATE_CACHE |
| 22 | #ifdef USE_PTHREAD_PSHARED |
| 23 | #include <pthread.h> |
| 24 | #else |
| 25 | #ifdef USE_SYSCALL_FUTEX |
| 26 | #include <unistd.h> |
| 27 | #include <linux/futex.h> |
| 28 | #include <sys/syscall.h> |
Emeric Brun | 3e541d1 | 2012-09-03 11:14:36 +0200 | [diff] [blame] | 29 | #endif |
Emeric Brun | 3e541d1 | 2012-09-03 11:14:36 +0200 | [diff] [blame] | 30 | #endif |
Willy Tarreau | ca36771 | 2021-06-15 15:03:19 +0200 | [diff] [blame] | 31 | #else |
| 32 | #include <haproxy/thread.h> |
Emeric Brun | 786991e | 2012-11-26 18:37:12 +0100 | [diff] [blame] | 33 | #endif |
| 34 | |
Frédéric Lécaille | b7838af | 2018-10-22 16:21:39 +0200 | [diff] [blame] | 35 | int shctx_init(struct shared_context **orig_shctx, |
Frédéric Lécaille | b80bc27 | 2018-10-25 20:31:40 +0200 | [diff] [blame] | 36 | int maxblocks, int blocksize, unsigned int maxobjsz, |
| 37 | int extra, int shared); |
Frédéric Lécaille | 0bec807 | 2018-10-22 17:55:57 +0200 | [diff] [blame] | 38 | struct shared_block *shctx_row_reserve_hot(struct shared_context *shctx, |
| 39 | struct shared_block *last, int data_len); |
William Lallemand | 4f45bb9 | 2017-10-30 20:08:51 +0100 | [diff] [blame] | 40 | void shctx_row_inc_hot(struct shared_context *shctx, struct shared_block *first); |
| 41 | void shctx_row_dec_hot(struct shared_context *shctx, struct shared_block *first); |
| 42 | int shctx_row_data_append(struct shared_context *shctx, |
Frédéric Lécaille | 0bec807 | 2018-10-22 17:55:57 +0200 | [diff] [blame] | 43 | struct shared_block *first, struct shared_block *from, |
| 44 | unsigned char *data, int len); |
William Lallemand | 4f45bb9 | 2017-10-30 20:08:51 +0100 | [diff] [blame] | 45 | int shctx_row_data_get(struct shared_context *shctx, struct shared_block *first, |
| 46 | unsigned char *dst, int offset, int len); |
Emeric Brun | caa19cc | 2014-05-07 16:10:18 +0200 | [diff] [blame] | 47 | |
William Lallemand | ed0b5ad | 2017-10-30 19:36:36 +0100 | [diff] [blame] | 48 | |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 49 | /* Lock functions */ |
| 50 | |
| 51 | #if defined (USE_PRIVATE_CACHE) |
Willy Tarreau | ca36771 | 2021-06-15 15:03:19 +0200 | [diff] [blame] | 52 | extern int use_shared_mem; |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 53 | |
Willy Tarreau | ca36771 | 2021-06-15 15:03:19 +0200 | [diff] [blame] | 54 | #define shctx_lock(shctx) if (use_shared_mem) HA_SPIN_LOCK(SHCTX_LOCK, &shctx->lock) |
| 55 | #define shctx_unlock(shctx) if (use_shared_mem) HA_SPIN_UNLOCK(SHCTX_LOCK, &shctx->lock) |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 56 | |
| 57 | #elif defined (USE_PTHREAD_PSHARED) |
| 58 | extern int use_shared_mem; |
| 59 | |
William Lallemand | a3c77cf | 2017-10-30 23:44:40 +0100 | [diff] [blame] | 60 | #define shctx_lock(shctx) if (use_shared_mem) pthread_mutex_lock(&shctx->mutex) |
| 61 | #define shctx_unlock(shctx) if (use_shared_mem) pthread_mutex_unlock(&shctx->mutex) |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 62 | |
| 63 | #else |
| 64 | extern int use_shared_mem; |
| 65 | |
| 66 | #ifdef USE_SYSCALL_FUTEX |
William Lallemand | a3c77cf | 2017-10-30 23:44:40 +0100 | [diff] [blame] | 67 | static inline void _shctx_wait4lock(unsigned int *count, unsigned int *uaddr, int value) |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 68 | { |
| 69 | syscall(SYS_futex, uaddr, FUTEX_WAIT, value, NULL, 0, 0); |
| 70 | } |
| 71 | |
William Lallemand | a3c77cf | 2017-10-30 23:44:40 +0100 | [diff] [blame] | 72 | static inline void _shctx_awakelocker(unsigned int *uaddr) |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 73 | { |
| 74 | syscall(SYS_futex, uaddr, FUTEX_WAKE, 1, NULL, 0, 0); |
| 75 | } |
| 76 | |
| 77 | #else /* internal spin lock */ |
| 78 | |
| 79 | #if defined (__i486__) || defined (__i586__) || defined (__i686__) || defined (__x86_64__) |
| 80 | static inline void relax() |
| 81 | { |
| 82 | __asm volatile("rep;nop\n" ::: "memory"); |
| 83 | } |
| 84 | #else /* if no x86_64 or i586 arch: use less optimized but generic asm */ |
| 85 | static inline void relax() |
| 86 | { |
| 87 | __asm volatile("" ::: "memory"); |
| 88 | } |
| 89 | #endif |
| 90 | |
William Lallemand | a3c77cf | 2017-10-30 23:44:40 +0100 | [diff] [blame] | 91 | static inline void _shctx_wait4lock(unsigned int *count, unsigned int *uaddr, int value) |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 92 | { |
| 93 | int i; |
| 94 | |
| 95 | for (i = 0; i < *count; i++) { |
| 96 | relax(); |
| 97 | relax(); |
Willy Tarreau | 3801bdc | 2020-05-01 13:05:29 +0200 | [diff] [blame] | 98 | if (*uaddr != value) |
| 99 | return; |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 100 | } |
Willy Tarreau | 86c6a92 | 2020-05-01 13:15:32 +0200 | [diff] [blame] | 101 | *count = (unsigned char)((*count << 1) + 1); |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 102 | } |
| 103 | |
William Lallemand | a3c77cf | 2017-10-30 23:44:40 +0100 | [diff] [blame] | 104 | #define _shctx_awakelocker(a) |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 105 | |
| 106 | #endif |
| 107 | |
| 108 | #if defined (__i486__) || defined (__i586__) || defined (__i686__) || defined (__x86_64__) |
| 109 | static inline unsigned int xchg(unsigned int *ptr, unsigned int x) |
| 110 | { |
| 111 | __asm volatile("lock xchgl %0,%1" |
| 112 | : "=r" (x), "+m" (*ptr) |
| 113 | : "0" (x) |
| 114 | : "memory"); |
| 115 | return x; |
| 116 | } |
| 117 | |
| 118 | static inline unsigned int cmpxchg(unsigned int *ptr, unsigned int old, unsigned int new) |
| 119 | { |
| 120 | unsigned int ret; |
| 121 | |
| 122 | __asm volatile("lock cmpxchgl %2,%1" |
| 123 | : "=a" (ret), "+m" (*ptr) |
| 124 | : "r" (new), "0" (old) |
| 125 | : "memory"); |
| 126 | return ret; |
| 127 | } |
| 128 | |
| 129 | static inline unsigned char atomic_dec(unsigned int *ptr) |
| 130 | { |
| 131 | unsigned char ret; |
| 132 | __asm volatile("lock decl %0\n" |
| 133 | "setne %1\n" |
| 134 | : "+m" (*ptr), "=qm" (ret) |
| 135 | : |
| 136 | : "memory"); |
| 137 | return ret; |
| 138 | } |
| 139 | |
| 140 | #else /* if no x86_64 or i586 arch: use less optimized gcc >= 4.1 built-ins */ |
| 141 | static inline unsigned int xchg(unsigned int *ptr, unsigned int x) |
| 142 | { |
| 143 | return __sync_lock_test_and_set(ptr, x); |
| 144 | } |
| 145 | |
| 146 | static inline unsigned int cmpxchg(unsigned int *ptr, unsigned int old, unsigned int new) |
| 147 | { |
| 148 | return __sync_val_compare_and_swap(ptr, old, new); |
| 149 | } |
| 150 | |
| 151 | static inline unsigned char atomic_dec(unsigned int *ptr) |
| 152 | { |
| 153 | return __sync_sub_and_fetch(ptr, 1) ? 1 : 0; |
| 154 | } |
| 155 | |
| 156 | #endif |
| 157 | |
William Lallemand | a3c77cf | 2017-10-30 23:44:40 +0100 | [diff] [blame] | 158 | static inline void _shctx_lock(struct shared_context *shctx) |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 159 | { |
| 160 | unsigned int x; |
Willy Tarreau | 86c6a92 | 2020-05-01 13:15:32 +0200 | [diff] [blame] | 161 | unsigned int count = 3; |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 162 | |
| 163 | x = cmpxchg(&shctx->waiters, 0, 1); |
| 164 | if (x) { |
| 165 | if (x != 2) |
| 166 | x = xchg(&shctx->waiters, 2); |
| 167 | |
| 168 | while (x) { |
William Lallemand | a3c77cf | 2017-10-30 23:44:40 +0100 | [diff] [blame] | 169 | _shctx_wait4lock(&count, &shctx->waiters, 2); |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 170 | x = xchg(&shctx->waiters, 2); |
| 171 | } |
| 172 | } |
| 173 | } |
| 174 | |
William Lallemand | a3c77cf | 2017-10-30 23:44:40 +0100 | [diff] [blame] | 175 | static inline void _shctx_unlock(struct shared_context *shctx) |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 176 | { |
| 177 | if (atomic_dec(&shctx->waiters)) { |
| 178 | shctx->waiters = 0; |
William Lallemand | a3c77cf | 2017-10-30 23:44:40 +0100 | [diff] [blame] | 179 | _shctx_awakelocker(&shctx->waiters); |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 180 | } |
| 181 | } |
| 182 | |
William Lallemand | a3c77cf | 2017-10-30 23:44:40 +0100 | [diff] [blame] | 183 | #define shctx_lock(shctx) if (use_shared_mem) _shctx_lock(shctx) |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 184 | |
William Lallemand | a3c77cf | 2017-10-30 23:44:40 +0100 | [diff] [blame] | 185 | #define shctx_unlock(shctx) if (use_shared_mem) _shctx_unlock(shctx) |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 186 | |
| 187 | #endif |
| 188 | |
William Lallemand | ed0b5ad | 2017-10-30 19:36:36 +0100 | [diff] [blame] | 189 | /* List Macros */ |
| 190 | |
Frédéric Lécaille | 0bec807 | 2018-10-22 17:55:57 +0200 | [diff] [blame] | 191 | /* |
| 192 | * Insert <s> block after <head> which is not necessarily the head of a list, |
| 193 | * so between <head> and the next element after <head>. |
| 194 | */ |
| 195 | static inline void shctx_block_append_hot(struct shared_context *shctx, |
| 196 | struct list *head, |
| 197 | struct shared_block *s) |
| 198 | { |
| 199 | shctx->nbav--; |
Willy Tarreau | 2b71810 | 2021-04-21 07:32:39 +0200 | [diff] [blame] | 200 | LIST_DELETE(&s->list); |
| 201 | LIST_INSERT(head, &s->list); |
Frédéric Lécaille | 0bec807 | 2018-10-22 17:55:57 +0200 | [diff] [blame] | 202 | } |
| 203 | |
William Lallemand | 4f45bb9 | 2017-10-30 20:08:51 +0100 | [diff] [blame] | 204 | static inline void shctx_block_set_hot(struct shared_context *shctx, |
William Lallemand | ed0b5ad | 2017-10-30 19:36:36 +0100 | [diff] [blame] | 205 | struct shared_block *s) |
| 206 | { |
William Lallemand | 4f45bb9 | 2017-10-30 20:08:51 +0100 | [diff] [blame] | 207 | shctx->nbav--; |
Willy Tarreau | 2b71810 | 2021-04-21 07:32:39 +0200 | [diff] [blame] | 208 | LIST_DELETE(&s->list); |
| 209 | LIST_APPEND(&shctx->hot, &s->list); |
William Lallemand | ed0b5ad | 2017-10-30 19:36:36 +0100 | [diff] [blame] | 210 | } |
| 211 | |
William Lallemand | 4f45bb9 | 2017-10-30 20:08:51 +0100 | [diff] [blame] | 212 | static inline void shctx_block_set_avail(struct shared_context *shctx, |
William Lallemand | ed0b5ad | 2017-10-30 19:36:36 +0100 | [diff] [blame] | 213 | struct shared_block *s) |
| 214 | { |
William Lallemand | 4f45bb9 | 2017-10-30 20:08:51 +0100 | [diff] [blame] | 215 | shctx->nbav++; |
Willy Tarreau | 2b71810 | 2021-04-21 07:32:39 +0200 | [diff] [blame] | 216 | LIST_DELETE(&s->list); |
| 217 | LIST_APPEND(&shctx->avail, &s->list); |
William Lallemand | ed0b5ad | 2017-10-30 19:36:36 +0100 | [diff] [blame] | 218 | } |
William Lallemand | 24a7a75 | 2017-10-09 14:17:39 +0200 | [diff] [blame] | 219 | |
Willy Tarreau | 334099c | 2020-06-03 18:38:48 +0200 | [diff] [blame] | 220 | #endif /* __HAPROXY_SHCTX_H */ |
Emeric Brun | 3e541d1 | 2012-09-03 11:14:36 +0200 | [diff] [blame] | 221 | |