blob: 01b218165eecaf504a3215e88cd8ab4cb4c1c88c [file] [log] [blame]
Christopher Faulet1a2b56e2017-10-12 16:09:09 +02001/*
2 * include/common/hathreads.h
3 * definitions, macros and inline functions about threads.
4 *
5 * Copyright (C) 2017 Christopher Fauet - cfaulet@haproxy.com
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation, version 2.1
10 * exclusively.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22#ifndef _COMMON_HATHREADS_H
23#define _COMMON_HATHREADS_H
24
25#include <common/config.h>
26
Willy Tarreau0ccd3222018-07-30 10:34:35 +020027/* Note about all_threads_mask :
28 * - with threads support disabled, this symbol is defined as zero (0UL).
29 * - with threads enabled, this variable is never zero, it contains the mask
30 * of enabled threads. Thus if only one thread is enabled, it equals 1.
31 */
32
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020033#ifndef USE_THREAD
34
Willy Tarreau421f02e2018-01-20 18:19:22 +010035#define MAX_THREADS 1
Willy Tarreau0c026f42018-08-01 19:12:20 +020036#define MAX_THREADS_MASK 1
37
38/* Only way found to replace variables with constants that are optimized away
39 * at build time.
40 */
41enum { all_threads_mask = 1UL };
42enum { tid_bit = 1UL };
43enum { tid = 0 };
Willy Tarreau421f02e2018-01-20 18:19:22 +010044
Christopher Faulet9dcf9b62017-11-13 10:34:01 +010045#define __decl_hathreads(decl)
46
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020047#define HA_ATOMIC_CAS(val, old, new) ({((*val) == (*old)) ? (*(val) = (new) , 1) : (*(old) = *(val), 0);})
48#define HA_ATOMIC_ADD(val, i) ({*(val) += (i);})
49#define HA_ATOMIC_SUB(val, i) ({*(val) -= (i);})
Willy Tarreau9378df82018-09-05 16:11:03 +020050#define HA_ATOMIC_XADD(val, i) \
51 ({ \
52 typeof((val)) __p_xadd = (val); \
53 typeof(*(val)) __old_xadd = *__p_xadd; \
54 *__p_xadd += i; \
55 __old_xadd; \
56 })
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020057#define HA_ATOMIC_AND(val, flags) ({*(val) &= (flags);})
58#define HA_ATOMIC_OR(val, flags) ({*(val) |= (flags);})
59#define HA_ATOMIC_XCHG(val, new) \
60 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020061 typeof(*(val)) __old_xchg = *(val); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020062 *(val) = new; \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020063 __old_xchg; \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020064 })
Willy Tarreau5266b3e2018-01-25 17:43:58 +010065#define HA_ATOMIC_BTS(val, bit) \
66 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020067 typeof((val)) __p_bts = (val); \
68 typeof(*__p_bts) __b_bts = (1UL << (bit)); \
69 typeof(*__p_bts) __t_bts = *__p_bts & __b_bts; \
70 if (!__t_bts) \
71 *__p_bts |= __b_bts; \
72 __t_bts; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +010073 })
74#define HA_ATOMIC_BTR(val, bit) \
75 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020076 typeof((val)) __p_btr = (val); \
77 typeof(*__p_btr) __b_btr = (1UL << (bit)); \
78 typeof(*__p_btr) __t_btr = *__p_btr & __b_btr; \
79 if (__t_btr) \
80 *__p_btr &= ~__b_btr; \
81 __t_btr; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +010082 })
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020083#define HA_ATOMIC_STORE(val, new) ({*(val) = new;})
84#define HA_ATOMIC_UPDATE_MAX(val, new) \
85 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020086 typeof(*(val)) __new_max = (new); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020087 \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020088 if (*(val) < __new_max) \
89 *(val) = __new_max; \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020090 *(val); \
91 })
92
93#define HA_ATOMIC_UPDATE_MIN(val, new) \
94 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020095 typeof(*(val)) __new_min = (new); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020096 \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020097 if (*(val) > __new_min) \
98 *(val) = __new_min; \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020099 *(val); \
100 })
101
Willy Tarreaub29dc952017-10-31 18:00:20 +0100102#define HA_BARRIER() do { } while (0)
Christopher Faulet339fff82017-10-19 11:59:15 +0200103
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100104#define HA_SPIN_INIT(l) do { /* do nothing */ } while(0)
105#define HA_SPIN_DESTROY(l) do { /* do nothing */ } while(0)
106#define HA_SPIN_LOCK(lbl, l) do { /* do nothing */ } while(0)
107#define HA_SPIN_TRYLOCK(lbl, l) ({ 0; })
108#define HA_SPIN_UNLOCK(lbl, l) do { /* do nothing */ } while(0)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200109
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100110#define HA_RWLOCK_INIT(l) do { /* do nothing */ } while(0)
111#define HA_RWLOCK_DESTROY(l) do { /* do nothing */ } while(0)
112#define HA_RWLOCK_WRLOCK(lbl, l) do { /* do nothing */ } while(0)
113#define HA_RWLOCK_TRYWRLOCK(lbl, l) ({ 0; })
114#define HA_RWLOCK_WRUNLOCK(lbl, l) do { /* do nothing */ } while(0)
115#define HA_RWLOCK_RDLOCK(lbl, l) do { /* do nothing */ } while(0)
116#define HA_RWLOCK_TRYRDLOCK(lbl, l) ({ 0; })
117#define HA_RWLOCK_RDUNLOCK(lbl, l) do { /* do nothing */ } while(0)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200118
William Lallemand6e1796e2018-06-07 11:23:40 +0200119#define ha_sigmask(how, set, oldset) sigprocmask(how, set, oldset)
120
Willy Tarreau0c026f42018-08-01 19:12:20 +0200121static inline void ha_set_tid(unsigned int tid)
122{
123}
William Lallemand6e1796e2018-06-07 11:23:40 +0200124
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100125static inline void __ha_barrier_load(void)
126{
127}
128
129static inline void __ha_barrier_store(void)
130{
131}
132
133static inline void __ha_barrier_full(void)
134{
135}
136
Willy Tarreau60b639c2018-08-02 10:16:17 +0200137static inline void thread_harmless_now()
138{
139}
140
141static inline void thread_harmless_end()
142{
143}
144
145static inline void thread_isolate()
146{
147}
148
149static inline void thread_release()
150{
151}
152
153static inline unsigned long thread_isolated()
154{
155 return 1;
156}
157
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200158#else /* USE_THREAD */
159
160#include <stdio.h>
161#include <stdlib.h>
162#include <string.h>
163#include <pthread.h>
164#include <import/plock.h>
165
Willy Tarreau421f02e2018-01-20 18:19:22 +0100166#define MAX_THREADS LONGBITS
Willy Tarreau0c026f42018-08-01 19:12:20 +0200167#define MAX_THREADS_MASK ((unsigned long)-1)
Willy Tarreau421f02e2018-01-20 18:19:22 +0100168
Christopher Faulet9dcf9b62017-11-13 10:34:01 +0100169#define __decl_hathreads(decl) decl
170
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200171/* TODO: thread: For now, we rely on GCC builtins but it could be a good idea to
172 * have a header file regrouping all functions dealing with threads. */
Willy Tarreau1a69af62018-01-04 18:49:31 +0100173
David Carlierec5e8452018-01-11 14:20:43 +0000174#if defined(__GNUC__) && (__GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ < 7) && !defined(__clang__)
Willy Tarreau1a69af62018-01-04 18:49:31 +0100175/* gcc < 4.7 */
176
177#define HA_ATOMIC_ADD(val, i) __sync_add_and_fetch(val, i)
178#define HA_ATOMIC_SUB(val, i) __sync_sub_and_fetch(val, i)
Willy Tarreau9378df82018-09-05 16:11:03 +0200179#define HA_ATOMIC_XADD(val, i) __sync_fetch_and_add(val, i)
Willy Tarreau1a69af62018-01-04 18:49:31 +0100180#define HA_ATOMIC_AND(val, flags) __sync_and_and_fetch(val, flags)
181#define HA_ATOMIC_OR(val, flags) __sync_or_and_fetch(val, flags)
182
183/* the CAS is a bit complicated. The older API doesn't support returning the
184 * value and the swap's result at the same time. So here we take what looks
185 * like the safest route, consisting in using the boolean version guaranteeing
186 * that the operation was performed or not, and we snoop a previous value. If
187 * the compare succeeds, we return. If it fails, we return the previous value,
188 * but only if it differs from the expected one. If it's the same it's a race
189 * thus we try again to avoid confusing a possibly sensitive caller.
190 */
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200191#define HA_ATOMIC_CAS(val, old, new) \
192 ({ \
193 typeof((val)) __val_cas = (val); \
194 typeof((old)) __oldp_cas = (old); \
195 typeof(*(old)) __oldv_cas; \
196 typeof((new)) __new_cas = (new); \
197 int __ret_cas; \
198 do { \
199 __oldv_cas = *__val_cas; \
200 __ret_cas = __sync_bool_compare_and_swap(__val_cas, *__oldp_cas, __new_cas); \
201 } while (!__ret_cas && *__oldp_cas == __oldv_cas); \
202 if (!__ret_cas) \
203 *__oldp_cas = __oldv_cas; \
204 __ret_cas; \
Willy Tarreau1a69af62018-01-04 18:49:31 +0100205 })
206
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200207#define HA_ATOMIC_XCHG(val, new) \
208 ({ \
209 typeof((val)) __val_xchg = (val); \
210 typeof(*(val)) __old_xchg; \
211 typeof((new)) __new_xchg = (new); \
212 do { __old_xchg = *__val_xchg; \
213 } while (!__sync_bool_compare_and_swap(__val_xchg, __old_xchg, __new_xchg)); \
214 __old_xchg; \
Willy Tarreau1a69af62018-01-04 18:49:31 +0100215 })
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100216
217#define HA_ATOMIC_BTS(val, bit) \
218 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200219 typeof(*(val)) __b_bts = (1UL << (bit)); \
220 __sync_fetch_and_or((val), __b_bts) & __b_bts; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100221 })
222
223#define HA_ATOMIC_BTR(val, bit) \
224 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200225 typeof(*(val)) __b_btr = (1UL << (bit)); \
226 __sync_fetch_and_and((val), ~__b_btr) & __b_btr; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100227 })
228
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200229#define HA_ATOMIC_STORE(val, new) \
230 ({ \
231 typeof((val)) __val_store = (val); \
232 typeof(*(val)) __old_store; \
233 typeof((new)) __new_store = (new); \
234 do { __old_store = *__val_store; \
235 } while (!__sync_bool_compare_and_swap(__val_store, __old_store, __new_store)); \
Willy Tarreau1a69af62018-01-04 18:49:31 +0100236 })
237#else
238/* gcc >= 4.7 */
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200239#define HA_ATOMIC_CAS(val, old, new) __atomic_compare_exchange_n(val, old, new, 0, 0, 0)
240#define HA_ATOMIC_ADD(val, i) __atomic_add_fetch(val, i, 0)
Willy Tarreau9378df82018-09-05 16:11:03 +0200241#define HA_ATOMIC_XADD(val, i) __atomic_fetch_add(val, i, 0)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200242#define HA_ATOMIC_SUB(val, i) __atomic_sub_fetch(val, i, 0)
243#define HA_ATOMIC_AND(val, flags) __atomic_and_fetch(val, flags, 0)
244#define HA_ATOMIC_OR(val, flags) __atomic_or_fetch(val, flags, 0)
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100245#define HA_ATOMIC_BTS(val, bit) \
246 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200247 typeof(*(val)) __b_bts = (1UL << (bit)); \
248 __sync_fetch_and_or((val), __b_bts) & __b_bts; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100249 })
250
251#define HA_ATOMIC_BTR(val, bit) \
252 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200253 typeof(*(val)) __b_btr = (1UL << (bit)); \
254 __sync_fetch_and_and((val), ~__b_btr) & __b_btr; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100255 })
256
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200257#define HA_ATOMIC_XCHG(val, new) __atomic_exchange_n(val, new, 0)
258#define HA_ATOMIC_STORE(val, new) __atomic_store_n(val, new, 0)
Willy Tarreau1a69af62018-01-04 18:49:31 +0100259#endif
260
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200261#define HA_ATOMIC_UPDATE_MAX(val, new) \
262 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200263 typeof(*(val)) __old_max = *(val); \
264 typeof(*(val)) __new_max = (new); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200265 \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200266 while (__old_max < __new_max && \
267 !HA_ATOMIC_CAS(val, &__old_max, __new_max)); \
268 *(val); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200269 })
270#define HA_ATOMIC_UPDATE_MIN(val, new) \
271 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200272 typeof(*(val)) __old_min = *(val); \
273 typeof(*(val)) __new_min = (new); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200274 \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200275 while (__old_min > __new_min && \
276 !HA_ATOMIC_CAS(val, &__old_min, __new_min)); \
277 *(val); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200278 })
279
Willy Tarreaub29dc952017-10-31 18:00:20 +0100280#define HA_BARRIER() pl_barrier()
281
Willy Tarreau60b639c2018-08-02 10:16:17 +0200282void thread_harmless_till_end();
283void thread_isolate();
284void thread_release();
Christopher Faulet339fff82017-10-19 11:59:15 +0200285
Willy Tarreau0c026f42018-08-01 19:12:20 +0200286extern THREAD_LOCAL unsigned int tid; /* The thread id */
287extern THREAD_LOCAL unsigned long tid_bit; /* The bit corresponding to the thread id */
Christopher Fauletddb6c162018-07-20 09:31:53 +0200288extern volatile unsigned long all_threads_mask;
Willy Tarreau60b639c2018-08-02 10:16:17 +0200289extern volatile unsigned long threads_want_rdv_mask;
290extern volatile unsigned long threads_harmless_mask;
291
292/* explanation for threads_want_rdv_mask and threads_harmless_mask :
293 * - threads_want_rdv_mask is a bit field indicating all threads that have
294 * requested a rendez-vous of other threads using thread_isolate().
295 * - threads_harmless_mask is a bit field indicating all threads that are
296 * currently harmless in that they promise not to access a shared resource.
297 *
298 * For a given thread, its bits in want_rdv and harmless can be translated like
299 * this :
300 *
301 * ----------+----------+----------------------------------------------------
302 * want_rdv | harmless | description
303 * ----------+----------+----------------------------------------------------
304 * 0 | 0 | thread not interested in RDV, possibly harmful
305 * 0 | 1 | thread not interested in RDV but harmless
306 * 1 | 1 | thread interested in RDV and waiting for its turn
307 * 1 | 0 | thread currently working isolated from others
308 * ----------+----------+----------------------------------------------------
309 */
Olivier Houchard6b96f722018-04-25 16:58:25 +0200310
William Lallemand6e1796e2018-06-07 11:23:40 +0200311#define ha_sigmask(how, set, oldset) pthread_sigmask(how, set, oldset)
312
Willy Tarreau0c026f42018-08-01 19:12:20 +0200313/* sets the thread ID and the TID bit for the current thread */
314static inline void ha_set_tid(unsigned int data)
315{
316 tid = data;
317 tid_bit = (1UL << tid);
318}
319
Willy Tarreau60b639c2018-08-02 10:16:17 +0200320/* Marks the thread as harmless. Note: this must be true, i.e. the thread must
321 * not be touching any unprotected shared resource during this period. Usually
322 * this is called before poll(), but it may also be placed around very slow
323 * calls (eg: some crypto operations). Needs to be terminated using
324 * thread_harmless_end().
325 */
326static inline void thread_harmless_now()
327{
328 HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
329}
330
331/* Ends the harmless period started by thread_harmless_now(). Usually this is
332 * placed after the poll() call. If it is discovered that a job was running and
333 * is relying on the thread still being harmless, the thread waits for the
334 * other one to finish.
335 */
336static inline void thread_harmless_end()
337{
338 while (1) {
339 HA_ATOMIC_AND(&threads_harmless_mask, ~tid_bit);
340 if (likely((threads_want_rdv_mask & all_threads_mask) == 0))
341 break;
342 thread_harmless_till_end();
343 }
344}
345
346/* an isolated thread has harmless cleared and want_rdv set */
347static inline unsigned long thread_isolated()
348{
349 return threads_want_rdv_mask & ~threads_harmless_mask & tid_bit;
350}
351
William Lallemand6e1796e2018-06-07 11:23:40 +0200352
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200353#if defined(DEBUG_THREAD) || defined(DEBUG_FULL)
354
Christopher Fauletf51bac22018-01-30 11:04:29 +0100355/* WARNING!!! if you update this enum, please also keep lock_label() up to date below */
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200356enum lock_label {
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200357 FD_LOCK,
Emeric Brunc60def82017-09-27 14:59:38 +0200358 TASK_RQ_LOCK,
359 TASK_WQ_LOCK,
Christopher Fauletb349e482017-08-29 09:52:38 +0200360 POOL_LOCK,
Christopher Faulet8d8aa0d2017-05-30 15:36:50 +0200361 LISTENER_LOCK,
362 LISTENER_QUEUE_LOCK,
Christopher Fauletff8abcd2017-06-02 15:33:24 +0200363 PROXY_LOCK,
Christopher Faulet29f77e82017-06-08 14:04:45 +0200364 SERVER_LOCK,
Christopher Faulet5b517552017-06-09 14:17:53 +0200365 LBPRM_LOCK,
Christopher Fauletb79a94c2017-05-30 15:34:30 +0200366 SIGNALS_LOCK,
Emeric Brun819fc6f2017-06-13 19:37:32 +0200367 STK_TABLE_LOCK,
368 STK_SESS_LOCK,
Emeric Brun1138fd02017-06-19 12:38:55 +0200369 APPLETS_LOCK,
Emeric Brun80527f52017-06-19 17:46:37 +0200370 PEER_LOCK,
Emeric Bruna1dd2432017-06-21 15:42:52 +0200371 BUF_WQ_LOCK,
Emeric Brun6b35e9b2017-06-30 16:23:45 +0200372 STRMS_LOCK,
Emeric Brun821bb9b2017-06-15 16:37:39 +0200373 SSL_LOCK,
374 SSL_GEN_CERTS_LOCK,
Emeric Brunb5997f72017-07-03 11:34:05 +0200375 PATREF_LOCK,
376 PATEXP_LOCK,
377 PATLRU_LOCK,
Christopher Faulete95f2c32017-07-24 16:30:34 +0200378 VARS_LOCK,
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +0200379 COMP_POOL_LOCK,
Thierry FOURNIER61ba0e22017-07-12 11:41:21 +0200380 LUA_LOCK,
Thierry FOURNIER738a6d72017-07-17 00:14:07 +0200381 NOTIF_LOCK,
Christopher Faulet24289f22017-09-25 14:48:02 +0200382 SPOE_APPLET_LOCK,
Christopher Fauletb2812a62017-10-04 16:17:58 +0200383 DNS_LOCK,
Christopher Fauletcfda8472017-10-20 15:40:23 +0200384 PID_LIST_LOCK,
Christopher Fauletc2a89a62017-10-23 15:54:24 +0200385 EMAIL_ALERTS_LOCK,
Emeric Brund8b3b652017-11-07 11:19:48 +0100386 PIPES_LOCK,
Willy Tarreau1605c7a2018-01-23 19:01:49 +0100387 START_LOCK,
Christopher Faulet16f45c82018-02-16 11:23:49 +0100388 TLSKEYS_REF_LOCK,
Christopher Faulet339fff82017-10-19 11:59:15 +0200389 LOCK_LABELS
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200390};
391struct lock_stat {
392 uint64_t nsec_wait_for_write;
393 uint64_t nsec_wait_for_read;
394 uint64_t num_write_locked;
395 uint64_t num_write_unlocked;
396 uint64_t num_read_locked;
397 uint64_t num_read_unlocked;
398};
399
400extern struct lock_stat lock_stats[LOCK_LABELS];
401
402#define __HA_SPINLOCK_T unsigned long
403
404#define __SPIN_INIT(l) ({ (*l) = 0; })
405#define __SPIN_DESTROY(l) ({ (*l) = 0; })
Willy Tarreau88ac59b2017-11-06 01:03:26 +0100406#define __SPIN_LOCK(l) pl_take_s(l)
407#define __SPIN_TRYLOCK(l) !pl_try_s(l)
408#define __SPIN_UNLOCK(l) pl_drop_s(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200409
410#define __HA_RWLOCK_T unsigned long
411
412#define __RWLOCK_INIT(l) ({ (*l) = 0; })
413#define __RWLOCK_DESTROY(l) ({ (*l) = 0; })
414#define __RWLOCK_WRLOCK(l) pl_take_w(l)
415#define __RWLOCK_TRYWRLOCK(l) !pl_try_w(l)
416#define __RWLOCK_WRUNLOCK(l) pl_drop_w(l)
417#define __RWLOCK_RDLOCK(l) pl_take_r(l)
418#define __RWLOCK_TRYRDLOCK(l) !pl_try_r(l)
419#define __RWLOCK_RDUNLOCK(l) pl_drop_r(l)
420
421#define HA_SPINLOCK_T struct ha_spinlock
422
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100423#define HA_SPIN_INIT(l) __spin_init(l)
424#define HA_SPIN_DESTROY(l) __spin_destroy(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200425
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100426#define HA_SPIN_LOCK(lbl, l) __spin_lock(lbl, l, __func__, __FILE__, __LINE__)
427#define HA_SPIN_TRYLOCK(lbl, l) __spin_trylock(lbl, l, __func__, __FILE__, __LINE__)
428#define HA_SPIN_UNLOCK(lbl, l) __spin_unlock(lbl, l, __func__, __FILE__, __LINE__)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200429
430#define HA_RWLOCK_T struct ha_rwlock
431
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100432#define HA_RWLOCK_INIT(l) __ha_rwlock_init((l))
433#define HA_RWLOCK_DESTROY(l) __ha_rwlock_destroy((l))
434#define HA_RWLOCK_WRLOCK(lbl,l) __ha_rwlock_wrlock(lbl, l, __func__, __FILE__, __LINE__)
435#define HA_RWLOCK_TRYWRLOCK(lbl,l) __ha_rwlock_trywrlock(lbl, l, __func__, __FILE__, __LINE__)
436#define HA_RWLOCK_WRUNLOCK(lbl,l) __ha_rwlock_wrunlock(lbl, l, __func__, __FILE__, __LINE__)
437#define HA_RWLOCK_RDLOCK(lbl,l) __ha_rwlock_rdlock(lbl, l)
438#define HA_RWLOCK_TRYRDLOCK(lbl,l) __ha_rwlock_tryrdlock(lbl, l)
439#define HA_RWLOCK_RDUNLOCK(lbl,l) __ha_rwlock_rdunlock(lbl, l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200440
441struct ha_spinlock {
442 __HA_SPINLOCK_T lock;
443 struct {
444 unsigned long owner; /* a bit is set to 1 << tid for the lock owner */
445 unsigned long waiters; /* a bit is set to 1 << tid for waiting threads */
446 struct {
447 const char *function;
448 const char *file;
449 int line;
450 } last_location; /* location of the last owner */
451 } info;
452};
453
454struct ha_rwlock {
455 __HA_RWLOCK_T lock;
456 struct {
457 unsigned long cur_writer; /* a bit is set to 1 << tid for the lock owner */
458 unsigned long wait_writers; /* a bit is set to 1 << tid for waiting writers */
459 unsigned long cur_readers; /* a bit is set to 1 << tid for current readers */
460 unsigned long wait_readers; /* a bit is set to 1 << tid for waiting waiters */
461 struct {
462 const char *function;
463 const char *file;
464 int line;
465 } last_location; /* location of the last write owner */
466 } info;
467};
468
Christopher Fauletf51bac22018-01-30 11:04:29 +0100469static inline const char *lock_label(enum lock_label label)
470{
471 switch (label) {
Christopher Fauletf51bac22018-01-30 11:04:29 +0100472 case FD_LOCK: return "FD";
473 case TASK_RQ_LOCK: return "TASK_RQ";
474 case TASK_WQ_LOCK: return "TASK_WQ";
475 case POOL_LOCK: return "POOL";
476 case LISTENER_LOCK: return "LISTENER";
477 case LISTENER_QUEUE_LOCK: return "LISTENER_QUEUE";
478 case PROXY_LOCK: return "PROXY";
479 case SERVER_LOCK: return "SERVER";
Christopher Fauletf51bac22018-01-30 11:04:29 +0100480 case LBPRM_LOCK: return "LBPRM";
481 case SIGNALS_LOCK: return "SIGNALS";
482 case STK_TABLE_LOCK: return "STK_TABLE";
483 case STK_SESS_LOCK: return "STK_SESS";
484 case APPLETS_LOCK: return "APPLETS";
485 case PEER_LOCK: return "PEER";
486 case BUF_WQ_LOCK: return "BUF_WQ";
487 case STRMS_LOCK: return "STRMS";
488 case SSL_LOCK: return "SSL";
489 case SSL_GEN_CERTS_LOCK: return "SSL_GEN_CERTS";
490 case PATREF_LOCK: return "PATREF";
491 case PATEXP_LOCK: return "PATEXP";
492 case PATLRU_LOCK: return "PATLRU";
493 case VARS_LOCK: return "VARS";
494 case COMP_POOL_LOCK: return "COMP_POOL";
495 case LUA_LOCK: return "LUA";
496 case NOTIF_LOCK: return "NOTIF";
497 case SPOE_APPLET_LOCK: return "SPOE_APPLET";
498 case DNS_LOCK: return "DNS";
499 case PID_LIST_LOCK: return "PID_LIST";
500 case EMAIL_ALERTS_LOCK: return "EMAIL_ALERTS";
501 case PIPES_LOCK: return "PIPES";
502 case START_LOCK: return "START";
Christopher Faulet16f45c82018-02-16 11:23:49 +0100503 case TLSKEYS_REF_LOCK: return "TLSKEYS_REF";
Christopher Fauletf51bac22018-01-30 11:04:29 +0100504 case LOCK_LABELS: break; /* keep compiler happy */
505 };
506 /* only way to come here is consecutive to an internal bug */
507 abort();
508}
509
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200510static inline void show_lock_stats()
511{
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200512 int lbl;
513
514 for (lbl = 0; lbl < LOCK_LABELS; lbl++) {
515 fprintf(stderr,
516 "Stats about Lock %s: \n"
517 "\t # write lock : %lu\n"
518 "\t # write unlock: %lu (%ld)\n"
519 "\t # wait time for write : %.3f msec\n"
520 "\t # wait time for write/lock: %.3f nsec\n"
521 "\t # read lock : %lu\n"
522 "\t # read unlock : %lu (%ld)\n"
523 "\t # wait time for read : %.3f msec\n"
524 "\t # wait time for read/lock : %.3f nsec\n",
Christopher Fauletf51bac22018-01-30 11:04:29 +0100525 lock_label(lbl),
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200526 lock_stats[lbl].num_write_locked,
527 lock_stats[lbl].num_write_unlocked,
528 lock_stats[lbl].num_write_unlocked - lock_stats[lbl].num_write_locked,
529 (double)lock_stats[lbl].nsec_wait_for_write / 1000000.0,
530 lock_stats[lbl].num_write_locked ? ((double)lock_stats[lbl].nsec_wait_for_write / (double)lock_stats[lbl].num_write_locked) : 0,
531 lock_stats[lbl].num_read_locked,
532 lock_stats[lbl].num_read_unlocked,
533 lock_stats[lbl].num_read_unlocked - lock_stats[lbl].num_read_locked,
534 (double)lock_stats[lbl].nsec_wait_for_read / 1000000.0,
535 lock_stats[lbl].num_read_locked ? ((double)lock_stats[lbl].nsec_wait_for_read / (double)lock_stats[lbl].num_read_locked) : 0);
536 }
537}
538
539/* Following functions are used to collect some stats about locks. We wrap
540 * pthread functions to known how much time we wait in a lock. */
541
542static uint64_t nsec_now(void) {
543 struct timespec ts;
544
545 clock_gettime(CLOCK_MONOTONIC, &ts);
546 return ((uint64_t) ts.tv_sec * 1000000000ULL +
547 (uint64_t) ts.tv_nsec);
548}
549
550static inline void __ha_rwlock_init(struct ha_rwlock *l)
551{
552 memset(l, 0, sizeof(struct ha_rwlock));
553 __RWLOCK_INIT(&l->lock);
554}
555
556static inline void __ha_rwlock_destroy(struct ha_rwlock *l)
557{
558 __RWLOCK_DESTROY(&l->lock);
559 memset(l, 0, sizeof(struct ha_rwlock));
560}
561
562
563static inline void __ha_rwlock_wrlock(enum lock_label lbl, struct ha_rwlock *l,
564 const char *func, const char *file, int line)
565{
566 uint64_t start_time;
567
568 if (unlikely(l->info.cur_writer & tid_bit)) {
569 /* the thread is already owning the lock for write */
570 abort();
571 }
572
573 if (unlikely(l->info.cur_readers & tid_bit)) {
574 /* the thread is already owning the lock for read */
575 abort();
576 }
577
578 HA_ATOMIC_OR(&l->info.wait_writers, tid_bit);
579
580 start_time = nsec_now();
581 __RWLOCK_WRLOCK(&l->lock);
582 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_write, (nsec_now() - start_time));
583
584 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
585
586 l->info.cur_writer = tid_bit;
587 l->info.last_location.function = func;
588 l->info.last_location.file = file;
589 l->info.last_location.line = line;
590
591 HA_ATOMIC_AND(&l->info.wait_writers, ~tid_bit);
592}
593
594static inline int __ha_rwlock_trywrlock(enum lock_label lbl, struct ha_rwlock *l,
595 const char *func, const char *file, int line)
596{
597 uint64_t start_time;
598 int r;
599
600 if (unlikely(l->info.cur_writer & tid_bit)) {
601 /* the thread is already owning the lock for write */
602 abort();
603 }
604
605 if (unlikely(l->info.cur_readers & tid_bit)) {
606 /* the thread is already owning the lock for read */
607 abort();
608 }
609
610 /* We set waiting writer because trywrlock could wait for readers to quit */
611 HA_ATOMIC_OR(&l->info.wait_writers, tid_bit);
612
613 start_time = nsec_now();
614 r = __RWLOCK_TRYWRLOCK(&l->lock);
615 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_write, (nsec_now() - start_time));
616 if (unlikely(r)) {
617 HA_ATOMIC_AND(&l->info.wait_writers, ~tid_bit);
618 return r;
619 }
620 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
621
622 l->info.cur_writer = tid_bit;
623 l->info.last_location.function = func;
624 l->info.last_location.file = file;
625 l->info.last_location.line = line;
626
627 HA_ATOMIC_AND(&l->info.wait_writers, ~tid_bit);
628
629 return 0;
630}
631
632static inline void __ha_rwlock_wrunlock(enum lock_label lbl,struct ha_rwlock *l,
633 const char *func, const char *file, int line)
634{
635 if (unlikely(!(l->info.cur_writer & tid_bit))) {
636 /* the thread is not owning the lock for write */
637 abort();
638 }
639
640 l->info.cur_writer = 0;
641 l->info.last_location.function = func;
642 l->info.last_location.file = file;
643 l->info.last_location.line = line;
644
645 __RWLOCK_WRUNLOCK(&l->lock);
646
647 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_unlocked, 1);
648}
649
650static inline void __ha_rwlock_rdlock(enum lock_label lbl,struct ha_rwlock *l)
651{
652 uint64_t start_time;
653
654 if (unlikely(l->info.cur_writer & tid_bit)) {
655 /* the thread is already owning the lock for write */
656 abort();
657 }
658
659 if (unlikely(l->info.cur_readers & tid_bit)) {
660 /* the thread is already owning the lock for read */
661 abort();
662 }
663
664 HA_ATOMIC_OR(&l->info.wait_readers, tid_bit);
665
666 start_time = nsec_now();
667 __RWLOCK_RDLOCK(&l->lock);
668 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_read, (nsec_now() - start_time));
669 HA_ATOMIC_ADD(&lock_stats[lbl].num_read_locked, 1);
670
671 HA_ATOMIC_OR(&l->info.cur_readers, tid_bit);
672
673 HA_ATOMIC_AND(&l->info.wait_readers, ~tid_bit);
674}
675
676static inline int __ha_rwlock_tryrdlock(enum lock_label lbl,struct ha_rwlock *l)
677{
678 int r;
679
680 if (unlikely(l->info.cur_writer & tid_bit)) {
681 /* the thread is already owning the lock for write */
682 abort();
683 }
684
685 if (unlikely(l->info.cur_readers & tid_bit)) {
686 /* the thread is already owning the lock for read */
687 abort();
688 }
689
690 /* try read should never wait */
691 r = __RWLOCK_TRYRDLOCK(&l->lock);
692 if (unlikely(r))
693 return r;
694 HA_ATOMIC_ADD(&lock_stats[lbl].num_read_locked, 1);
695
696 HA_ATOMIC_OR(&l->info.cur_readers, tid_bit);
697
698 return 0;
699}
700
701static inline void __ha_rwlock_rdunlock(enum lock_label lbl,struct ha_rwlock *l)
702{
703 if (unlikely(!(l->info.cur_readers & tid_bit))) {
704 /* the thread is not owning the lock for read */
705 abort();
706 }
707
708 HA_ATOMIC_AND(&l->info.cur_readers, ~tid_bit);
709
710 __RWLOCK_RDUNLOCK(&l->lock);
711
712 HA_ATOMIC_ADD(&lock_stats[lbl].num_read_unlocked, 1);
713}
714
715static inline void __spin_init(struct ha_spinlock *l)
716{
717 memset(l, 0, sizeof(struct ha_spinlock));
718 __SPIN_INIT(&l->lock);
719}
720
721static inline void __spin_destroy(struct ha_spinlock *l)
722{
723 __SPIN_DESTROY(&l->lock);
724 memset(l, 0, sizeof(struct ha_spinlock));
725}
726
727static inline void __spin_lock(enum lock_label lbl, struct ha_spinlock *l,
728 const char *func, const char *file, int line)
729{
730 uint64_t start_time;
731
732 if (unlikely(l->info.owner & tid_bit)) {
733 /* the thread is already owning the lock */
734 abort();
735 }
736
737 HA_ATOMIC_OR(&l->info.waiters, tid_bit);
738
739 start_time = nsec_now();
740 __SPIN_LOCK(&l->lock);
741 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_write, (nsec_now() - start_time));
742
743 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
744
745
746 l->info.owner = tid_bit;
747 l->info.last_location.function = func;
748 l->info.last_location.file = file;
749 l->info.last_location.line = line;
750
751 HA_ATOMIC_AND(&l->info.waiters, ~tid_bit);
752}
753
754static inline int __spin_trylock(enum lock_label lbl, struct ha_spinlock *l,
755 const char *func, const char *file, int line)
756{
757 int r;
758
759 if (unlikely(l->info.owner & tid_bit)) {
760 /* the thread is already owning the lock */
761 abort();
762 }
763
764 /* try read should never wait */
765 r = __SPIN_TRYLOCK(&l->lock);
766 if (unlikely(r))
767 return r;
768 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
769
770 l->info.owner = tid_bit;
771 l->info.last_location.function = func;
772 l->info.last_location.file = file;
773 l->info.last_location.line = line;
774
775 return 0;
776}
777
778static inline void __spin_unlock(enum lock_label lbl, struct ha_spinlock *l,
779 const char *func, const char *file, int line)
780{
781 if (unlikely(!(l->info.owner & tid_bit))) {
782 /* the thread is not owning the lock */
783 abort();
784 }
785
786 l->info.owner = 0;
787 l->info.last_location.function = func;
788 l->info.last_location.file = file;
789 l->info.last_location.line = line;
790
Willy Tarreau7c2a2ad2017-11-02 16:26:02 +0100791 __SPIN_UNLOCK(&l->lock);
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200792 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_unlocked, 1);
793}
794
795#else /* DEBUG_THREAD */
796
797#define HA_SPINLOCK_T unsigned long
798
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100799#define HA_SPIN_INIT(l) ({ (*l) = 0; })
800#define HA_SPIN_DESTROY(l) ({ (*l) = 0; })
801#define HA_SPIN_LOCK(lbl, l) pl_take_s(l)
802#define HA_SPIN_TRYLOCK(lbl, l) !pl_try_s(l)
803#define HA_SPIN_UNLOCK(lbl, l) pl_drop_s(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200804
805#define HA_RWLOCK_T unsigned long
806
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100807#define HA_RWLOCK_INIT(l) ({ (*l) = 0; })
808#define HA_RWLOCK_DESTROY(l) ({ (*l) = 0; })
809#define HA_RWLOCK_WRLOCK(lbl,l) pl_take_w(l)
810#define HA_RWLOCK_TRYWRLOCK(lbl,l) !pl_try_w(l)
811#define HA_RWLOCK_WRUNLOCK(lbl,l) pl_drop_w(l)
812#define HA_RWLOCK_RDLOCK(lbl,l) pl_take_r(l)
813#define HA_RWLOCK_TRYRDLOCK(lbl,l) !pl_try_r(l)
814#define HA_RWLOCK_RDUNLOCK(lbl,l) pl_drop_r(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200815
816#endif /* DEBUG_THREAD */
817
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100818#ifdef __x86_64__
819#define HA_HAVE_CAS_DW 1
820#define HA_CAS_IS_8B
821static __inline int
822__ha_cas_dw(void *target, void *compare, const void *set)
823{
824 char ret;
825
826 __asm __volatile("lock cmpxchg16b %0; setz %3"
827 : "+m" (*(void **)target),
828 "=a" (((void **)compare)[0]),
829 "=d" (((void **)compare)[1]),
830 "=q" (ret)
831 : "a" (((void **)compare)[0]),
832 "d" (((void **)compare)[1]),
833 "b" (((const void **)set)[0]),
834 "c" (((const void **)set)[1])
835 : "memory", "cc");
836 return (ret);
837}
838
839static __inline void
840__ha_barrier_load(void)
841{
842 __asm __volatile("lfence" ::: "memory");
843}
844
845static __inline void
846__ha_barrier_store(void)
847{
848 __asm __volatile("sfence" ::: "memory");
849}
850
851static __inline void
852__ha_barrier_full(void)
853{
854 __asm __volatile("mfence" ::: "memory");
855}
856
857#elif defined(__arm__) && (defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__))
858#define HA_HAVE_CAS_DW 1
859static __inline void
860__ha_barrier_load(void)
861{
862 __asm __volatile("dmb" ::: "memory");
863}
864
865static __inline void
866__ha_barrier_store(void)
867{
868 __asm __volatile("dsb" ::: "memory");
869}
870
871static __inline void
872__ha_barrier_full(void)
873{
874 __asm __volatile("dmb" ::: "memory");
875}
876
Willy Tarreau41ccb192018-02-14 14:16:28 +0100877static __inline int __ha_cas_dw(void *target, void *compare, const void *set)
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100878{
879 uint64_t previous;
880 int tmp;
881
882 __asm __volatile("1:"
883 "ldrexd %0, [%4];"
884 "cmp %Q0, %Q2;"
885 "ittt eq;"
886 "cmpeq %R0, %R2;"
887 "strexdeq %1, %3, [%4];"
888 "cmpeq %1, #1;"
889 "beq 1b;"
890 : "=&r" (previous), "=&r" (tmp)
Willy Tarreau41ccb192018-02-14 14:16:28 +0100891 : "r" (*(uint64_t *)compare), "r" (*(uint64_t *)set), "r" (target)
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100892 : "memory", "cc");
893 tmp = (previous == *(uint64_t *)compare);
894 *(uint64_t *)compare = previous;
895 return (tmp);
896}
897
898#elif defined (__aarch64__)
899#define HA_HAVE_CAS_DW 1
900#define HA_CAS_IS_8B
901
902static __inline void
903__ha_barrier_load(void)
904{
905 __asm __volatile("dmb ishld" ::: "memory");
906}
907
908static __inline void
909__ha_barrier_store(void)
910{
911 __asm __volatile("dmb ishst" ::: "memory");
912}
913
914static __inline void
915__ha_barrier_full(void)
916{
917 __asm __volatile("dmb ish" ::: "memory");
918}
919
920static __inline int __ha_cas_dw(void *target, void *compare, void *set)
921{
922 void *value[2];
923 uint64_t tmp1, tmp2;
924
925 __asm__ __volatile__("1:"
926 "ldxp %0, %1, [%4];"
927 "mov %2, %0;"
928 "mov %3, %1;"
929 "eor %0, %0, %5;"
930 "eor %1, %1, %6;"
931 "orr %1, %0, %1;"
932 "mov %w0, #0;"
933 "cbnz %1, 2f;"
934 "stxp %w0, %7, %8, [%4];"
935 "cbnz %w0, 1b;"
936 "mov %w0, #1;"
937 "2:"
938 : "=&r" (tmp1), "=&r" (tmp2), "=&r" (value[0]), "=&r" (value[1])
939 : "r" (target), "r" (((void **)(compare))[0]), "r" (((void **)(compare))[1]), "r" (((void **)(set))[0]), "r" (((void **)(set))[1])
940 : "cc", "memory");
941
942 memcpy(compare, &value, sizeof(value));
943 return (tmp1);
944}
945
946#else
947#define __ha_barrier_load __sync_synchronize
948#define __ha_barrier_store __sync_synchronize
949#define __ha_barrier_full __sync_synchronize
950#endif
951
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200952#endif /* USE_THREAD */
953
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100954static inline void __ha_compiler_barrier(void)
955{
956 __asm __volatile("" ::: "memory");
957}
958
Willy Tarreau0ccd3222018-07-30 10:34:35 +0200959int parse_nbthread(const char *arg, char **err);
Willy Tarreau4037a3f2018-03-28 18:06:47 +0200960
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200961#endif /* _COMMON_HATHREADS_H */