blob: a7c8dc936da06efeb548de7b05f1f34f2d99fc15 [file] [log] [blame]
Christopher Faulet1a2b56e2017-10-12 16:09:09 +02001/*
2 * include/common/hathreads.h
3 * definitions, macros and inline functions about threads.
4 *
5 * Copyright (C) 2017 Christopher Fauet - cfaulet@haproxy.com
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation, version 2.1
10 * exclusively.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22#ifndef _COMMON_HATHREADS_H
23#define _COMMON_HATHREADS_H
24
Willy Tarreau2beaaf72019-05-22 08:43:34 +020025#include <signal.h>
Willy Tarreau38171da2019-05-17 16:33:13 +020026#include <unistd.h>
27#ifdef _POSIX_PRIORITY_SCHEDULING
28#include <sched.h>
29#endif
30
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020031#include <common/config.h>
Willy Tarreau90fa97b2018-11-25 19:46:08 +010032#include <common/initcall.h>
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020033
Willy Tarreau0ccd3222018-07-30 10:34:35 +020034/* Note about all_threads_mask :
Willy Tarreauda9e9392019-02-02 17:03:41 +010035 * - this variable is comprised between 1 and LONGBITS.
36 * - with threads support disabled, this symbol is defined as constant 1UL.
37 * - with threads enabled, it contains the mask of enabled threads. Thus if
38 * only one thread is enabled, it equals 1.
Willy Tarreau0ccd3222018-07-30 10:34:35 +020039 */
40
Willy Tarreaue6a02fa2019-05-22 07:06:44 +020041/* thread info flags, for thread_info[].flags */
42#define TI_FL_STUCK 0x00000001
43
44
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020045#ifndef USE_THREAD
46
Willy Tarreau421f02e2018-01-20 18:19:22 +010047#define MAX_THREADS 1
Willy Tarreau0c026f42018-08-01 19:12:20 +020048#define MAX_THREADS_MASK 1
49
50/* Only way found to replace variables with constants that are optimized away
51 * at build time.
52 */
53enum { all_threads_mask = 1UL };
Willy Tarreau441259c2019-05-22 07:48:18 +020054enum { threads_harmless_mask = 0 };
55enum { threads_want_rdv_mask = 0 };
Willy Tarreau9a1f5732019-06-09 12:20:02 +020056enum { threads_sync_mask = 0 };
Willy Tarreau0c026f42018-08-01 19:12:20 +020057enum { tid_bit = 1UL };
58enum { tid = 0 };
Willy Tarreau421f02e2018-01-20 18:19:22 +010059
Willy Tarreau5a6e2242019-05-20 18:56:48 +020060extern struct thread_info {
Willy Tarreau624dcbf2019-05-20 20:23:06 +020061 clockid_t clock_id;
Willy Tarreau430f5902019-05-21 20:01:26 +020062 timer_t wd_timer; /* valid timer or TIMER_INVALID if not set */
Willy Tarreau81036f22019-05-20 19:24:50 +020063 uint64_t prev_cpu_time; /* previous per thread CPU time */
64 uint64_t prev_mono_time; /* previous system wide monotonic time */
65 unsigned int idle_pct; /* idle to total ratio over last sample (percent) */
Willy Tarreaue6a02fa2019-05-22 07:06:44 +020066 unsigned int flags; /* thread info flags, TI_FL_* */
Willy Tarreau5a6e2242019-05-20 18:56:48 +020067 /* pad to cache line (64B) */
68 char __pad[0]; /* unused except to check remaining room */
69 char __end[0] __attribute__((aligned(64)));
70} thread_info[MAX_THREADS];
71
Willy Tarreau8323a372019-05-20 18:57:53 +020072extern THREAD_LOCAL struct thread_info *ti; /* thread_info for the current thread */
73
Christopher Faulet9dcf9b62017-11-13 10:34:01 +010074#define __decl_hathreads(decl)
Willy Tarreau90fa97b2018-11-25 19:46:08 +010075#define __decl_spinlock(lock)
76#define __decl_aligned_spinlock(lock)
77#define __decl_rwlock(lock)
78#define __decl_aligned_rwlock(lock)
Christopher Faulet9dcf9b62017-11-13 10:34:01 +010079
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020080#define HA_ATOMIC_CAS(val, old, new) ({((*val) == (*old)) ? (*(val) = (new) , 1) : (*(old) = *(val), 0);})
Willy Tarreauc3b59582019-05-27 17:37:20 +020081
82/* warning, n is a pointer to the double value for dwcas */
83#define HA_ATOMIC_DWCAS(val, o, n) \
84 ({ \
85 long *_v = (long*)(val); \
86 long *_o = (long*)(o); \
87 long *_n = (long*)(n); \
88 long _v0 = _v[0], _v1 = _v[1]; \
89 (_v0 == _o[0] && _v1 == _o[1]) ? \
90 (_v[0] = _n[0], _v[1] = _n[1], 1) : \
91 (_o[0] = _v0, _o[1] = _v1, 0); \
92 })
93
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020094#define HA_ATOMIC_ADD(val, i) ({*(val) += (i);})
95#define HA_ATOMIC_SUB(val, i) ({*(val) -= (i);})
Willy Tarreau9378df82018-09-05 16:11:03 +020096#define HA_ATOMIC_XADD(val, i) \
97 ({ \
98 typeof((val)) __p_xadd = (val); \
99 typeof(*(val)) __old_xadd = *__p_xadd; \
100 *__p_xadd += i; \
101 __old_xadd; \
102 })
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200103#define HA_ATOMIC_AND(val, flags) ({*(val) &= (flags);})
104#define HA_ATOMIC_OR(val, flags) ({*(val) |= (flags);})
105#define HA_ATOMIC_XCHG(val, new) \
106 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200107 typeof(*(val)) __old_xchg = *(val); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200108 *(val) = new; \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200109 __old_xchg; \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200110 })
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100111#define HA_ATOMIC_BTS(val, bit) \
112 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200113 typeof((val)) __p_bts = (val); \
114 typeof(*__p_bts) __b_bts = (1UL << (bit)); \
115 typeof(*__p_bts) __t_bts = *__p_bts & __b_bts; \
116 if (!__t_bts) \
117 *__p_bts |= __b_bts; \
118 __t_bts; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100119 })
120#define HA_ATOMIC_BTR(val, bit) \
121 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200122 typeof((val)) __p_btr = (val); \
123 typeof(*__p_btr) __b_btr = (1UL << (bit)); \
124 typeof(*__p_btr) __t_btr = *__p_btr & __b_btr; \
125 if (__t_btr) \
126 *__p_btr &= ~__b_btr; \
127 __t_btr; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100128 })
Olivier Houchard9ce62b52019-04-30 13:38:02 +0200129#define HA_ATOMIC_LOAD(val) *(val)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200130#define HA_ATOMIC_STORE(val, new) ({*(val) = new;})
131#define HA_ATOMIC_UPDATE_MAX(val, new) \
132 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200133 typeof(*(val)) __new_max = (new); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200134 \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200135 if (*(val) < __new_max) \
136 *(val) = __new_max; \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200137 *(val); \
138 })
139
140#define HA_ATOMIC_UPDATE_MIN(val, new) \
141 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200142 typeof(*(val)) __new_min = (new); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200143 \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200144 if (*(val) > __new_min) \
145 *(val) = __new_min; \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200146 *(val); \
147 })
148
Willy Tarreaub29dc952017-10-31 18:00:20 +0100149#define HA_BARRIER() do { } while (0)
Christopher Faulet339fff82017-10-19 11:59:15 +0200150
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100151#define HA_SPIN_INIT(l) do { /* do nothing */ } while(0)
152#define HA_SPIN_DESTROY(l) do { /* do nothing */ } while(0)
153#define HA_SPIN_LOCK(lbl, l) do { /* do nothing */ } while(0)
154#define HA_SPIN_TRYLOCK(lbl, l) ({ 0; })
155#define HA_SPIN_UNLOCK(lbl, l) do { /* do nothing */ } while(0)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200156
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100157#define HA_RWLOCK_INIT(l) do { /* do nothing */ } while(0)
158#define HA_RWLOCK_DESTROY(l) do { /* do nothing */ } while(0)
159#define HA_RWLOCK_WRLOCK(lbl, l) do { /* do nothing */ } while(0)
160#define HA_RWLOCK_TRYWRLOCK(lbl, l) ({ 0; })
161#define HA_RWLOCK_WRUNLOCK(lbl, l) do { /* do nothing */ } while(0)
162#define HA_RWLOCK_RDLOCK(lbl, l) do { /* do nothing */ } while(0)
163#define HA_RWLOCK_TRYRDLOCK(lbl, l) ({ 0; })
164#define HA_RWLOCK_RDUNLOCK(lbl, l) do { /* do nothing */ } while(0)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200165
William Lallemand6e1796e2018-06-07 11:23:40 +0200166#define ha_sigmask(how, set, oldset) sigprocmask(how, set, oldset)
167
Willy Tarreau0c026f42018-08-01 19:12:20 +0200168static inline void ha_set_tid(unsigned int tid)
169{
Willy Tarreau8323a372019-05-20 18:57:53 +0200170 ti = &thread_info[tid];
Willy Tarreau38171da2019-05-17 16:33:13 +0200171}
172
173static inline void ha_thread_relax(void)
174{
175#if _POSIX_PRIORITY_SCHEDULING
176 sched_yield();
177#endif
Willy Tarreau0c026f42018-08-01 19:12:20 +0200178}
William Lallemand6e1796e2018-06-07 11:23:40 +0200179
Willy Tarreau2beaaf72019-05-22 08:43:34 +0200180/* send signal <sig> to thread <thr> */
181static inline void ha_tkill(unsigned int thr, int sig)
182{
183 raise(sig);
184}
185
186/* send signal <sig> to all threads */
187static inline void ha_tkillall(int sig)
188{
189 raise(sig);
190}
191
Olivier Houchard9abcf6e2019-03-07 18:45:00 +0100192static inline void __ha_barrier_atomic_load(void)
193{
194}
195
196static inline void __ha_barrier_atomic_store(void)
197{
198}
199
200static inline void __ha_barrier_atomic_full(void)
201{
202}
203
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100204static inline void __ha_barrier_load(void)
205{
206}
207
208static inline void __ha_barrier_store(void)
209{
210}
211
212static inline void __ha_barrier_full(void)
213{
214}
215
Willy Tarreau60b639c2018-08-02 10:16:17 +0200216static inline void thread_harmless_now()
217{
218}
219
220static inline void thread_harmless_end()
221{
222}
223
224static inline void thread_isolate()
225{
226}
227
228static inline void thread_release()
229{
230}
231
Willy Tarreau9a1f5732019-06-09 12:20:02 +0200232static inline void thread_sync_release()
233{
234}
235
Willy Tarreau60b639c2018-08-02 10:16:17 +0200236static inline unsigned long thread_isolated()
237{
238 return 1;
239}
240
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200241#else /* USE_THREAD */
242
243#include <stdio.h>
244#include <stdlib.h>
245#include <string.h>
246#include <pthread.h>
247#include <import/plock.h>
248
Willy Tarreauf5809cd2019-01-26 13:35:03 +0100249#ifndef MAX_THREADS
Willy Tarreau421f02e2018-01-20 18:19:22 +0100250#define MAX_THREADS LONGBITS
Willy Tarreauf5809cd2019-01-26 13:35:03 +0100251#endif
252
253#define MAX_THREADS_MASK (~0UL >> (LONGBITS - MAX_THREADS))
Willy Tarreau421f02e2018-01-20 18:19:22 +0100254
Christopher Faulet9dcf9b62017-11-13 10:34:01 +0100255#define __decl_hathreads(decl) decl
256
Willy Tarreau90fa97b2018-11-25 19:46:08 +0100257/* declare a self-initializing spinlock */
258#define __decl_spinlock(lock) \
259 HA_SPINLOCK_T (lock); \
260 INITCALL1(STG_LOCK, ha_spin_init, &(lock))
261
262/* declare a self-initializing spinlock, aligned on a cache line */
263#define __decl_aligned_spinlock(lock) \
264 HA_SPINLOCK_T (lock) __attribute__((aligned(64))); \
265 INITCALL1(STG_LOCK, ha_spin_init, &(lock))
266
267/* declare a self-initializing rwlock */
268#define __decl_rwlock(lock) \
269 HA_RWLOCK_T (lock); \
270 INITCALL1(STG_LOCK, ha_rwlock_init, &(lock))
271
272/* declare a self-initializing rwlock, aligned on a cache line */
273#define __decl_aligned_rwlock(lock) \
274 HA_RWLOCK_T (lock) __attribute__((aligned(64))); \
275 INITCALL1(STG_LOCK, ha_rwlock_init, &(lock))
276
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200277/* TODO: thread: For now, we rely on GCC builtins but it could be a good idea to
278 * have a header file regrouping all functions dealing with threads. */
Willy Tarreau1a69af62018-01-04 18:49:31 +0100279
David Carlierec5e8452018-01-11 14:20:43 +0000280#if defined(__GNUC__) && (__GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ < 7) && !defined(__clang__)
Willy Tarreau1a69af62018-01-04 18:49:31 +0100281/* gcc < 4.7 */
282
283#define HA_ATOMIC_ADD(val, i) __sync_add_and_fetch(val, i)
284#define HA_ATOMIC_SUB(val, i) __sync_sub_and_fetch(val, i)
Willy Tarreau9378df82018-09-05 16:11:03 +0200285#define HA_ATOMIC_XADD(val, i) __sync_fetch_and_add(val, i)
Willy Tarreau1a69af62018-01-04 18:49:31 +0100286#define HA_ATOMIC_AND(val, flags) __sync_and_and_fetch(val, flags)
287#define HA_ATOMIC_OR(val, flags) __sync_or_and_fetch(val, flags)
288
289/* the CAS is a bit complicated. The older API doesn't support returning the
290 * value and the swap's result at the same time. So here we take what looks
291 * like the safest route, consisting in using the boolean version guaranteeing
292 * that the operation was performed or not, and we snoop a previous value. If
293 * the compare succeeds, we return. If it fails, we return the previous value,
294 * but only if it differs from the expected one. If it's the same it's a race
295 * thus we try again to avoid confusing a possibly sensitive caller.
296 */
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200297#define HA_ATOMIC_CAS(val, old, new) \
298 ({ \
299 typeof((val)) __val_cas = (val); \
300 typeof((old)) __oldp_cas = (old); \
301 typeof(*(old)) __oldv_cas; \
302 typeof((new)) __new_cas = (new); \
303 int __ret_cas; \
304 do { \
305 __oldv_cas = *__val_cas; \
306 __ret_cas = __sync_bool_compare_and_swap(__val_cas, *__oldp_cas, __new_cas); \
307 } while (!__ret_cas && *__oldp_cas == __oldv_cas); \
308 if (!__ret_cas) \
309 *__oldp_cas = __oldv_cas; \
310 __ret_cas; \
Willy Tarreau1a69af62018-01-04 18:49:31 +0100311 })
312
Willy Tarreauc3b59582019-05-27 17:37:20 +0200313/* warning, n is a pointer to the double value for dwcas */
Willy Tarreau6a38b322019-05-11 18:04:24 +0200314#define HA_ATOMIC_DWCAS(val, o, n) __ha_cas_dw(val, o, n)
315
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200316#define HA_ATOMIC_XCHG(val, new) \
317 ({ \
318 typeof((val)) __val_xchg = (val); \
319 typeof(*(val)) __old_xchg; \
320 typeof((new)) __new_xchg = (new); \
321 do { __old_xchg = *__val_xchg; \
322 } while (!__sync_bool_compare_and_swap(__val_xchg, __old_xchg, __new_xchg)); \
323 __old_xchg; \
Willy Tarreau1a69af62018-01-04 18:49:31 +0100324 })
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100325
326#define HA_ATOMIC_BTS(val, bit) \
327 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200328 typeof(*(val)) __b_bts = (1UL << (bit)); \
329 __sync_fetch_and_or((val), __b_bts) & __b_bts; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100330 })
331
332#define HA_ATOMIC_BTR(val, bit) \
333 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200334 typeof(*(val)) __b_btr = (1UL << (bit)); \
335 __sync_fetch_and_and((val), ~__b_btr) & __b_btr; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100336 })
337
Olivier Houchard9ce62b52019-04-30 13:38:02 +0200338#define HA_ATOMIC_LOAD(val) \
339 ({ \
340 typeof(*(val)) ret; \
341 __sync_synchronize(); \
342 ret = *(volatile typeof(val))val; \
343 __sync_synchronize(); \
344 ret; \
345 })
346
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200347#define HA_ATOMIC_STORE(val, new) \
348 ({ \
349 typeof((val)) __val_store = (val); \
350 typeof(*(val)) __old_store; \
351 typeof((new)) __new_store = (new); \
352 do { __old_store = *__val_store; \
353 } while (!__sync_bool_compare_and_swap(__val_store, __old_store, __new_store)); \
Willy Tarreau1a69af62018-01-04 18:49:31 +0100354 })
355#else
356/* gcc >= 4.7 */
Olivier Houchard11353792019-03-07 18:48:22 +0100357#define HA_ATOMIC_CAS(val, old, new) __atomic_compare_exchange_n(val, old, new, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
Willy Tarreauc3b59582019-05-27 17:37:20 +0200358/* warning, n is a pointer to the double value for dwcas */
Willy Tarreau6a38b322019-05-11 18:04:24 +0200359#define HA_ATOMIC_DWCAS(val, o, n) __ha_cas_dw(val, o, n)
Olivier Houchard11353792019-03-07 18:48:22 +0100360#define HA_ATOMIC_ADD(val, i) __atomic_add_fetch(val, i, __ATOMIC_SEQ_CST)
361#define HA_ATOMIC_XADD(val, i) __atomic_fetch_add(val, i, __ATOMIC_SEQ_CST)
362#define HA_ATOMIC_SUB(val, i) __atomic_sub_fetch(val, i, __ATOMIC_SEQ_CST)
363#define HA_ATOMIC_AND(val, flags) __atomic_and_fetch(val, flags, __ATOMIC_SEQ_CST)
364#define HA_ATOMIC_OR(val, flags) __atomic_or_fetch(val, flags, __ATOMIC_SEQ_CST)
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100365#define HA_ATOMIC_BTS(val, bit) \
366 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200367 typeof(*(val)) __b_bts = (1UL << (bit)); \
368 __sync_fetch_and_or((val), __b_bts) & __b_bts; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100369 })
370
371#define HA_ATOMIC_BTR(val, bit) \
372 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200373 typeof(*(val)) __b_btr = (1UL << (bit)); \
374 __sync_fetch_and_and((val), ~__b_btr) & __b_btr; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100375 })
376
Olivier Houchard11353792019-03-07 18:48:22 +0100377#define HA_ATOMIC_XCHG(val, new) __atomic_exchange_n(val, new, __ATOMIC_SEQ_CST)
378#define HA_ATOMIC_STORE(val, new) __atomic_store_n(val, new, __ATOMIC_SEQ_CST)
Olivier Houchard9ce62b52019-04-30 13:38:02 +0200379#define HA_ATOMIC_LOAD(val) __atomic_load_n(val, __ATOMIC_SEQ_CST)
Olivier Houchard3212a2c2019-04-15 21:14:25 +0200380
381/* Variants that don't generate any memory barrier.
382 * If you're unsure how to deal with barriers, just use the HA_ATOMIC_* version,
383 * that will always generate correct code.
384 * Usually it's fine to use those when updating data that have no dependency,
385 * ie updating a counter. Otherwise a barrier is required.
386 */
387#define _HA_ATOMIC_CAS(val, old, new) __atomic_compare_exchange_n(val, old, new, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED)
Willy Tarreauc3b59582019-05-27 17:37:20 +0200388/* warning, n is a pointer to the double value for dwcas */
Willy Tarreau6a38b322019-05-11 18:04:24 +0200389#define _HA_ATOMIC_DWCAS(val, o, n) __ha_cas_dw(val, o, n)
Olivier Houchard3212a2c2019-04-15 21:14:25 +0200390#define _HA_ATOMIC_ADD(val, i) __atomic_add_fetch(val, i, __ATOMIC_RELAXED)
391#define _HA_ATOMIC_XADD(val, i) __atomic_fetch_add(val, i, __ATOMIC_RELAXED)
392#define _HA_ATOMIC_SUB(val, i) __atomic_sub_fetch(val, i, __ATOMIC_RELAXED)
393#define _HA_ATOMIC_AND(val, flags) __atomic_and_fetch(val, flags, __ATOMIC_RELAXED)
394#define _HA_ATOMIC_OR(val, flags) __atomic_or_fetch(val, flags, __ATOMIC_RELAXED)
395#define _HA_ATOMIC_XCHG(val, new) __atomic_exchange_n(val, new, __ATOMIC_RELAXED)
396#define _HA_ATOMIC_STORE(val, new) __atomic_store_n(val, new, __ATOMIC_RELAXED)
Olivier Houchard9ce62b52019-04-30 13:38:02 +0200397#define _HA_ATOMIC_LOAD(val) __atomic_load_n(val, __ATOMIC_RELAXED)
Olivier Houchard3212a2c2019-04-15 21:14:25 +0200398
399#endif /* gcc >= 4.7 */
Willy Tarreau1a69af62018-01-04 18:49:31 +0100400
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200401#define HA_ATOMIC_UPDATE_MAX(val, new) \
402 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200403 typeof(*(val)) __old_max = *(val); \
404 typeof(*(val)) __new_max = (new); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200405 \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200406 while (__old_max < __new_max && \
407 !HA_ATOMIC_CAS(val, &__old_max, __new_max)); \
408 *(val); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200409 })
410#define HA_ATOMIC_UPDATE_MIN(val, new) \
411 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200412 typeof(*(val)) __old_min = *(val); \
413 typeof(*(val)) __new_min = (new); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200414 \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200415 while (__old_min > __new_min && \
416 !HA_ATOMIC_CAS(val, &__old_min, __new_min)); \
417 *(val); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200418 })
419
Willy Tarreaub29dc952017-10-31 18:00:20 +0100420#define HA_BARRIER() pl_barrier()
421
Willy Tarreau60b639c2018-08-02 10:16:17 +0200422void thread_harmless_till_end();
423void thread_isolate();
424void thread_release();
Willy Tarreau9a1f5732019-06-09 12:20:02 +0200425void thread_sync_release();
Willy Tarreau2beaaf72019-05-22 08:43:34 +0200426void ha_tkill(unsigned int thr, int sig);
427void ha_tkillall(int sig);
Christopher Faulet339fff82017-10-19 11:59:15 +0200428
Willy Tarreau5a6e2242019-05-20 18:56:48 +0200429extern struct thread_info {
430 pthread_t pthread;
431 clockid_t clock_id;
Willy Tarreau430f5902019-05-21 20:01:26 +0200432 timer_t wd_timer; /* valid timer or TIMER_INVALID if not set */
Willy Tarreau81036f22019-05-20 19:24:50 +0200433 uint64_t prev_cpu_time; /* previous per thread CPU time */
434 uint64_t prev_mono_time; /* previous system wide monotonic time */
435 unsigned int idle_pct; /* idle to total ratio over last sample (percent) */
Willy Tarreaue6a02fa2019-05-22 07:06:44 +0200436 unsigned int flags; /* thread info flags, TI_FL_* */
Willy Tarreau5a6e2242019-05-20 18:56:48 +0200437 /* pad to cache line (64B) */
438 char __pad[0]; /* unused except to check remaining room */
439 char __end[0] __attribute__((aligned(64)));
440} thread_info[MAX_THREADS];
441
Willy Tarreau0c026f42018-08-01 19:12:20 +0200442extern THREAD_LOCAL unsigned int tid; /* The thread id */
443extern THREAD_LOCAL unsigned long tid_bit; /* The bit corresponding to the thread id */
Willy Tarreau8323a372019-05-20 18:57:53 +0200444extern THREAD_LOCAL struct thread_info *ti; /* thread_info for the current thread */
Christopher Fauletddb6c162018-07-20 09:31:53 +0200445extern volatile unsigned long all_threads_mask;
Willy Tarreau60b639c2018-08-02 10:16:17 +0200446extern volatile unsigned long threads_want_rdv_mask;
447extern volatile unsigned long threads_harmless_mask;
Willy Tarreau9a1f5732019-06-09 12:20:02 +0200448extern volatile unsigned long threads_sync_mask;
Willy Tarreau60b639c2018-08-02 10:16:17 +0200449
Willy Tarreau9a1f5732019-06-09 12:20:02 +0200450/* explanation for threads_want_rdv_mask, threads_harmless_mask, and
451 * threads_sync_mask :
Willy Tarreau60b639c2018-08-02 10:16:17 +0200452 * - threads_want_rdv_mask is a bit field indicating all threads that have
453 * requested a rendez-vous of other threads using thread_isolate().
454 * - threads_harmless_mask is a bit field indicating all threads that are
455 * currently harmless in that they promise not to access a shared resource.
Willy Tarreau9a1f5732019-06-09 12:20:02 +0200456 * - threads_sync_mask is a bit field indicating that a thread waiting for
457 * others to finish wants to leave synchronized with others and as such
458 * promises to do so as well using thread_sync_release().
Willy Tarreau60b639c2018-08-02 10:16:17 +0200459 *
460 * For a given thread, its bits in want_rdv and harmless can be translated like
461 * this :
462 *
463 * ----------+----------+----------------------------------------------------
464 * want_rdv | harmless | description
465 * ----------+----------+----------------------------------------------------
466 * 0 | 0 | thread not interested in RDV, possibly harmful
467 * 0 | 1 | thread not interested in RDV but harmless
468 * 1 | 1 | thread interested in RDV and waiting for its turn
469 * 1 | 0 | thread currently working isolated from others
470 * ----------+----------+----------------------------------------------------
Willy Tarreau9a1f5732019-06-09 12:20:02 +0200471 *
472 * thread_sync_mask only delays the leaving of threads_sync_release() to make
473 * sure that each thread's harmless bit is cleared before leaving the function.
Willy Tarreau60b639c2018-08-02 10:16:17 +0200474 */
Olivier Houchard6b96f722018-04-25 16:58:25 +0200475
William Lallemand6e1796e2018-06-07 11:23:40 +0200476#define ha_sigmask(how, set, oldset) pthread_sigmask(how, set, oldset)
477
Willy Tarreau0c026f42018-08-01 19:12:20 +0200478/* sets the thread ID and the TID bit for the current thread */
479static inline void ha_set_tid(unsigned int data)
480{
481 tid = data;
482 tid_bit = (1UL << tid);
Willy Tarreau8323a372019-05-20 18:57:53 +0200483 ti = &thread_info[tid];
Willy Tarreau0c026f42018-08-01 19:12:20 +0200484}
485
Willy Tarreau38171da2019-05-17 16:33:13 +0200486static inline void ha_thread_relax(void)
487{
488#if _POSIX_PRIORITY_SCHEDULING
489 sched_yield();
490#else
491 pl_cpu_relax();
492#endif
493}
494
Willy Tarreau60b639c2018-08-02 10:16:17 +0200495/* Marks the thread as harmless. Note: this must be true, i.e. the thread must
496 * not be touching any unprotected shared resource during this period. Usually
497 * this is called before poll(), but it may also be placed around very slow
498 * calls (eg: some crypto operations). Needs to be terminated using
499 * thread_harmless_end().
500 */
501static inline void thread_harmless_now()
502{
503 HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
504}
505
506/* Ends the harmless period started by thread_harmless_now(). Usually this is
507 * placed after the poll() call. If it is discovered that a job was running and
508 * is relying on the thread still being harmless, the thread waits for the
509 * other one to finish.
510 */
511static inline void thread_harmless_end()
512{
513 while (1) {
514 HA_ATOMIC_AND(&threads_harmless_mask, ~tid_bit);
515 if (likely((threads_want_rdv_mask & all_threads_mask) == 0))
516 break;
517 thread_harmless_till_end();
518 }
519}
520
521/* an isolated thread has harmless cleared and want_rdv set */
522static inline unsigned long thread_isolated()
523{
524 return threads_want_rdv_mask & ~threads_harmless_mask & tid_bit;
525}
526
William Lallemand6e1796e2018-06-07 11:23:40 +0200527
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200528#if defined(DEBUG_THREAD) || defined(DEBUG_FULL)
529
Christopher Fauletf51bac22018-01-30 11:04:29 +0100530/* WARNING!!! if you update this enum, please also keep lock_label() up to date below */
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200531enum lock_label {
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200532 FD_LOCK,
Emeric Brunc60def82017-09-27 14:59:38 +0200533 TASK_RQ_LOCK,
534 TASK_WQ_LOCK,
Christopher Fauletb349e482017-08-29 09:52:38 +0200535 POOL_LOCK,
Christopher Faulet8d8aa0d2017-05-30 15:36:50 +0200536 LISTENER_LOCK,
Christopher Fauletff8abcd2017-06-02 15:33:24 +0200537 PROXY_LOCK,
Christopher Faulet29f77e82017-06-08 14:04:45 +0200538 SERVER_LOCK,
Christopher Faulet5b517552017-06-09 14:17:53 +0200539 LBPRM_LOCK,
Christopher Fauletb79a94c2017-05-30 15:34:30 +0200540 SIGNALS_LOCK,
Emeric Brun819fc6f2017-06-13 19:37:32 +0200541 STK_TABLE_LOCK,
542 STK_SESS_LOCK,
Emeric Brun1138fd02017-06-19 12:38:55 +0200543 APPLETS_LOCK,
Emeric Brun80527f52017-06-19 17:46:37 +0200544 PEER_LOCK,
Emeric Bruna1dd2432017-06-21 15:42:52 +0200545 BUF_WQ_LOCK,
Emeric Brun6b35e9b2017-06-30 16:23:45 +0200546 STRMS_LOCK,
Emeric Brun821bb9b2017-06-15 16:37:39 +0200547 SSL_LOCK,
548 SSL_GEN_CERTS_LOCK,
Emeric Brunb5997f72017-07-03 11:34:05 +0200549 PATREF_LOCK,
550 PATEXP_LOCK,
551 PATLRU_LOCK,
Christopher Faulete95f2c32017-07-24 16:30:34 +0200552 VARS_LOCK,
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +0200553 COMP_POOL_LOCK,
Thierry FOURNIER61ba0e22017-07-12 11:41:21 +0200554 LUA_LOCK,
Thierry FOURNIER738a6d72017-07-17 00:14:07 +0200555 NOTIF_LOCK,
Christopher Faulet24289f22017-09-25 14:48:02 +0200556 SPOE_APPLET_LOCK,
Christopher Fauletb2812a62017-10-04 16:17:58 +0200557 DNS_LOCK,
Christopher Fauletcfda8472017-10-20 15:40:23 +0200558 PID_LIST_LOCK,
Christopher Fauletc2a89a62017-10-23 15:54:24 +0200559 EMAIL_ALERTS_LOCK,
Emeric Brund8b3b652017-11-07 11:19:48 +0100560 PIPES_LOCK,
Christopher Faulet16f45c82018-02-16 11:23:49 +0100561 TLSKEYS_REF_LOCK,
Willy Tarreau34d4b522018-10-29 18:02:54 +0100562 AUTH_LOCK,
Frédéric Lécailled803e472019-04-25 07:42:09 +0200563 LOGSRV_LOCK,
Frédéric Lécaille4a3fef82019-05-28 14:47:17 +0200564 DICT_LOCK,
Ben51Degrees4ddf59d2019-02-05 13:24:00 +0000565 OTHER_LOCK,
Christopher Faulet339fff82017-10-19 11:59:15 +0200566 LOCK_LABELS
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200567};
568struct lock_stat {
569 uint64_t nsec_wait_for_write;
570 uint64_t nsec_wait_for_read;
571 uint64_t num_write_locked;
572 uint64_t num_write_unlocked;
573 uint64_t num_read_locked;
574 uint64_t num_read_unlocked;
575};
576
577extern struct lock_stat lock_stats[LOCK_LABELS];
578
579#define __HA_SPINLOCK_T unsigned long
580
581#define __SPIN_INIT(l) ({ (*l) = 0; })
582#define __SPIN_DESTROY(l) ({ (*l) = 0; })
Willy Tarreau88ac59b2017-11-06 01:03:26 +0100583#define __SPIN_LOCK(l) pl_take_s(l)
584#define __SPIN_TRYLOCK(l) !pl_try_s(l)
585#define __SPIN_UNLOCK(l) pl_drop_s(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200586
587#define __HA_RWLOCK_T unsigned long
588
589#define __RWLOCK_INIT(l) ({ (*l) = 0; })
590#define __RWLOCK_DESTROY(l) ({ (*l) = 0; })
591#define __RWLOCK_WRLOCK(l) pl_take_w(l)
592#define __RWLOCK_TRYWRLOCK(l) !pl_try_w(l)
593#define __RWLOCK_WRUNLOCK(l) pl_drop_w(l)
594#define __RWLOCK_RDLOCK(l) pl_take_r(l)
595#define __RWLOCK_TRYRDLOCK(l) !pl_try_r(l)
596#define __RWLOCK_RDUNLOCK(l) pl_drop_r(l)
597
598#define HA_SPINLOCK_T struct ha_spinlock
599
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100600#define HA_SPIN_INIT(l) __spin_init(l)
601#define HA_SPIN_DESTROY(l) __spin_destroy(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200602
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100603#define HA_SPIN_LOCK(lbl, l) __spin_lock(lbl, l, __func__, __FILE__, __LINE__)
604#define HA_SPIN_TRYLOCK(lbl, l) __spin_trylock(lbl, l, __func__, __FILE__, __LINE__)
605#define HA_SPIN_UNLOCK(lbl, l) __spin_unlock(lbl, l, __func__, __FILE__, __LINE__)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200606
607#define HA_RWLOCK_T struct ha_rwlock
608
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100609#define HA_RWLOCK_INIT(l) __ha_rwlock_init((l))
610#define HA_RWLOCK_DESTROY(l) __ha_rwlock_destroy((l))
611#define HA_RWLOCK_WRLOCK(lbl,l) __ha_rwlock_wrlock(lbl, l, __func__, __FILE__, __LINE__)
612#define HA_RWLOCK_TRYWRLOCK(lbl,l) __ha_rwlock_trywrlock(lbl, l, __func__, __FILE__, __LINE__)
613#define HA_RWLOCK_WRUNLOCK(lbl,l) __ha_rwlock_wrunlock(lbl, l, __func__, __FILE__, __LINE__)
614#define HA_RWLOCK_RDLOCK(lbl,l) __ha_rwlock_rdlock(lbl, l)
615#define HA_RWLOCK_TRYRDLOCK(lbl,l) __ha_rwlock_tryrdlock(lbl, l)
616#define HA_RWLOCK_RDUNLOCK(lbl,l) __ha_rwlock_rdunlock(lbl, l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200617
618struct ha_spinlock {
619 __HA_SPINLOCK_T lock;
620 struct {
621 unsigned long owner; /* a bit is set to 1 << tid for the lock owner */
622 unsigned long waiters; /* a bit is set to 1 << tid for waiting threads */
623 struct {
624 const char *function;
625 const char *file;
626 int line;
627 } last_location; /* location of the last owner */
628 } info;
629};
630
631struct ha_rwlock {
632 __HA_RWLOCK_T lock;
633 struct {
634 unsigned long cur_writer; /* a bit is set to 1 << tid for the lock owner */
635 unsigned long wait_writers; /* a bit is set to 1 << tid for waiting writers */
636 unsigned long cur_readers; /* a bit is set to 1 << tid for current readers */
637 unsigned long wait_readers; /* a bit is set to 1 << tid for waiting waiters */
638 struct {
639 const char *function;
640 const char *file;
641 int line;
642 } last_location; /* location of the last write owner */
643 } info;
644};
645
Christopher Fauletf51bac22018-01-30 11:04:29 +0100646static inline const char *lock_label(enum lock_label label)
647{
648 switch (label) {
Christopher Fauletf51bac22018-01-30 11:04:29 +0100649 case FD_LOCK: return "FD";
650 case TASK_RQ_LOCK: return "TASK_RQ";
651 case TASK_WQ_LOCK: return "TASK_WQ";
652 case POOL_LOCK: return "POOL";
653 case LISTENER_LOCK: return "LISTENER";
Christopher Fauletf51bac22018-01-30 11:04:29 +0100654 case PROXY_LOCK: return "PROXY";
655 case SERVER_LOCK: return "SERVER";
Christopher Fauletf51bac22018-01-30 11:04:29 +0100656 case LBPRM_LOCK: return "LBPRM";
657 case SIGNALS_LOCK: return "SIGNALS";
658 case STK_TABLE_LOCK: return "STK_TABLE";
659 case STK_SESS_LOCK: return "STK_SESS";
660 case APPLETS_LOCK: return "APPLETS";
661 case PEER_LOCK: return "PEER";
662 case BUF_WQ_LOCK: return "BUF_WQ";
663 case STRMS_LOCK: return "STRMS";
664 case SSL_LOCK: return "SSL";
665 case SSL_GEN_CERTS_LOCK: return "SSL_GEN_CERTS";
666 case PATREF_LOCK: return "PATREF";
667 case PATEXP_LOCK: return "PATEXP";
668 case PATLRU_LOCK: return "PATLRU";
669 case VARS_LOCK: return "VARS";
670 case COMP_POOL_LOCK: return "COMP_POOL";
671 case LUA_LOCK: return "LUA";
672 case NOTIF_LOCK: return "NOTIF";
673 case SPOE_APPLET_LOCK: return "SPOE_APPLET";
674 case DNS_LOCK: return "DNS";
675 case PID_LIST_LOCK: return "PID_LIST";
676 case EMAIL_ALERTS_LOCK: return "EMAIL_ALERTS";
677 case PIPES_LOCK: return "PIPES";
Christopher Faulet16f45c82018-02-16 11:23:49 +0100678 case TLSKEYS_REF_LOCK: return "TLSKEYS_REF";
Willy Tarreau34d4b522018-10-29 18:02:54 +0100679 case AUTH_LOCK: return "AUTH";
Frédéric Lécailled803e472019-04-25 07:42:09 +0200680 case LOGSRV_LOCK: return "LOGSRV";
Frédéric Lécaille4a3fef82019-05-28 14:47:17 +0200681 case DICT_LOCK: return "DICT";
Ben51Degrees4ddf59d2019-02-05 13:24:00 +0000682 case OTHER_LOCK: return "OTHER";
Christopher Fauletf51bac22018-01-30 11:04:29 +0100683 case LOCK_LABELS: break; /* keep compiler happy */
684 };
685 /* only way to come here is consecutive to an internal bug */
686 abort();
687}
688
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200689static inline void show_lock_stats()
690{
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200691 int lbl;
692
693 for (lbl = 0; lbl < LOCK_LABELS; lbl++) {
694 fprintf(stderr,
695 "Stats about Lock %s: \n"
696 "\t # write lock : %lu\n"
697 "\t # write unlock: %lu (%ld)\n"
698 "\t # wait time for write : %.3f msec\n"
699 "\t # wait time for write/lock: %.3f nsec\n"
700 "\t # read lock : %lu\n"
701 "\t # read unlock : %lu (%ld)\n"
702 "\t # wait time for read : %.3f msec\n"
703 "\t # wait time for read/lock : %.3f nsec\n",
Christopher Fauletf51bac22018-01-30 11:04:29 +0100704 lock_label(lbl),
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200705 lock_stats[lbl].num_write_locked,
706 lock_stats[lbl].num_write_unlocked,
707 lock_stats[lbl].num_write_unlocked - lock_stats[lbl].num_write_locked,
708 (double)lock_stats[lbl].nsec_wait_for_write / 1000000.0,
709 lock_stats[lbl].num_write_locked ? ((double)lock_stats[lbl].nsec_wait_for_write / (double)lock_stats[lbl].num_write_locked) : 0,
710 lock_stats[lbl].num_read_locked,
711 lock_stats[lbl].num_read_unlocked,
712 lock_stats[lbl].num_read_unlocked - lock_stats[lbl].num_read_locked,
713 (double)lock_stats[lbl].nsec_wait_for_read / 1000000.0,
714 lock_stats[lbl].num_read_locked ? ((double)lock_stats[lbl].nsec_wait_for_read / (double)lock_stats[lbl].num_read_locked) : 0);
715 }
716}
717
718/* Following functions are used to collect some stats about locks. We wrap
719 * pthread functions to known how much time we wait in a lock. */
720
721static uint64_t nsec_now(void) {
722 struct timespec ts;
723
724 clock_gettime(CLOCK_MONOTONIC, &ts);
725 return ((uint64_t) ts.tv_sec * 1000000000ULL +
726 (uint64_t) ts.tv_nsec);
727}
728
729static inline void __ha_rwlock_init(struct ha_rwlock *l)
730{
731 memset(l, 0, sizeof(struct ha_rwlock));
732 __RWLOCK_INIT(&l->lock);
733}
734
735static inline void __ha_rwlock_destroy(struct ha_rwlock *l)
736{
737 __RWLOCK_DESTROY(&l->lock);
738 memset(l, 0, sizeof(struct ha_rwlock));
739}
740
741
742static inline void __ha_rwlock_wrlock(enum lock_label lbl, struct ha_rwlock *l,
743 const char *func, const char *file, int line)
744{
745 uint64_t start_time;
746
747 if (unlikely(l->info.cur_writer & tid_bit)) {
748 /* the thread is already owning the lock for write */
749 abort();
750 }
751
752 if (unlikely(l->info.cur_readers & tid_bit)) {
753 /* the thread is already owning the lock for read */
754 abort();
755 }
756
757 HA_ATOMIC_OR(&l->info.wait_writers, tid_bit);
758
759 start_time = nsec_now();
760 __RWLOCK_WRLOCK(&l->lock);
761 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_write, (nsec_now() - start_time));
762
763 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
764
765 l->info.cur_writer = tid_bit;
766 l->info.last_location.function = func;
767 l->info.last_location.file = file;
768 l->info.last_location.line = line;
769
770 HA_ATOMIC_AND(&l->info.wait_writers, ~tid_bit);
771}
772
773static inline int __ha_rwlock_trywrlock(enum lock_label lbl, struct ha_rwlock *l,
774 const char *func, const char *file, int line)
775{
776 uint64_t start_time;
777 int r;
778
779 if (unlikely(l->info.cur_writer & tid_bit)) {
780 /* the thread is already owning the lock for write */
781 abort();
782 }
783
784 if (unlikely(l->info.cur_readers & tid_bit)) {
785 /* the thread is already owning the lock for read */
786 abort();
787 }
788
789 /* We set waiting writer because trywrlock could wait for readers to quit */
790 HA_ATOMIC_OR(&l->info.wait_writers, tid_bit);
791
792 start_time = nsec_now();
793 r = __RWLOCK_TRYWRLOCK(&l->lock);
794 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_write, (nsec_now() - start_time));
795 if (unlikely(r)) {
796 HA_ATOMIC_AND(&l->info.wait_writers, ~tid_bit);
797 return r;
798 }
799 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
800
801 l->info.cur_writer = tid_bit;
802 l->info.last_location.function = func;
803 l->info.last_location.file = file;
804 l->info.last_location.line = line;
805
806 HA_ATOMIC_AND(&l->info.wait_writers, ~tid_bit);
807
808 return 0;
809}
810
811static inline void __ha_rwlock_wrunlock(enum lock_label lbl,struct ha_rwlock *l,
812 const char *func, const char *file, int line)
813{
814 if (unlikely(!(l->info.cur_writer & tid_bit))) {
815 /* the thread is not owning the lock for write */
816 abort();
817 }
818
819 l->info.cur_writer = 0;
820 l->info.last_location.function = func;
821 l->info.last_location.file = file;
822 l->info.last_location.line = line;
823
824 __RWLOCK_WRUNLOCK(&l->lock);
825
826 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_unlocked, 1);
827}
828
829static inline void __ha_rwlock_rdlock(enum lock_label lbl,struct ha_rwlock *l)
830{
831 uint64_t start_time;
832
833 if (unlikely(l->info.cur_writer & tid_bit)) {
834 /* the thread is already owning the lock for write */
835 abort();
836 }
837
838 if (unlikely(l->info.cur_readers & tid_bit)) {
839 /* the thread is already owning the lock for read */
840 abort();
841 }
842
843 HA_ATOMIC_OR(&l->info.wait_readers, tid_bit);
844
845 start_time = nsec_now();
846 __RWLOCK_RDLOCK(&l->lock);
847 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_read, (nsec_now() - start_time));
848 HA_ATOMIC_ADD(&lock_stats[lbl].num_read_locked, 1);
849
850 HA_ATOMIC_OR(&l->info.cur_readers, tid_bit);
851
852 HA_ATOMIC_AND(&l->info.wait_readers, ~tid_bit);
853}
854
855static inline int __ha_rwlock_tryrdlock(enum lock_label lbl,struct ha_rwlock *l)
856{
857 int r;
858
859 if (unlikely(l->info.cur_writer & tid_bit)) {
860 /* the thread is already owning the lock for write */
861 abort();
862 }
863
864 if (unlikely(l->info.cur_readers & tid_bit)) {
865 /* the thread is already owning the lock for read */
866 abort();
867 }
868
869 /* try read should never wait */
870 r = __RWLOCK_TRYRDLOCK(&l->lock);
871 if (unlikely(r))
872 return r;
873 HA_ATOMIC_ADD(&lock_stats[lbl].num_read_locked, 1);
874
875 HA_ATOMIC_OR(&l->info.cur_readers, tid_bit);
876
877 return 0;
878}
879
880static inline void __ha_rwlock_rdunlock(enum lock_label lbl,struct ha_rwlock *l)
881{
882 if (unlikely(!(l->info.cur_readers & tid_bit))) {
883 /* the thread is not owning the lock for read */
884 abort();
885 }
886
887 HA_ATOMIC_AND(&l->info.cur_readers, ~tid_bit);
888
889 __RWLOCK_RDUNLOCK(&l->lock);
890
891 HA_ATOMIC_ADD(&lock_stats[lbl].num_read_unlocked, 1);
892}
893
894static inline void __spin_init(struct ha_spinlock *l)
895{
896 memset(l, 0, sizeof(struct ha_spinlock));
897 __SPIN_INIT(&l->lock);
898}
899
900static inline void __spin_destroy(struct ha_spinlock *l)
901{
902 __SPIN_DESTROY(&l->lock);
903 memset(l, 0, sizeof(struct ha_spinlock));
904}
905
906static inline void __spin_lock(enum lock_label lbl, struct ha_spinlock *l,
907 const char *func, const char *file, int line)
908{
909 uint64_t start_time;
910
911 if (unlikely(l->info.owner & tid_bit)) {
912 /* the thread is already owning the lock */
913 abort();
914 }
915
916 HA_ATOMIC_OR(&l->info.waiters, tid_bit);
917
918 start_time = nsec_now();
919 __SPIN_LOCK(&l->lock);
920 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_write, (nsec_now() - start_time));
921
922 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
923
924
925 l->info.owner = tid_bit;
926 l->info.last_location.function = func;
927 l->info.last_location.file = file;
928 l->info.last_location.line = line;
929
930 HA_ATOMIC_AND(&l->info.waiters, ~tid_bit);
931}
932
933static inline int __spin_trylock(enum lock_label lbl, struct ha_spinlock *l,
934 const char *func, const char *file, int line)
935{
936 int r;
937
938 if (unlikely(l->info.owner & tid_bit)) {
939 /* the thread is already owning the lock */
940 abort();
941 }
942
943 /* try read should never wait */
944 r = __SPIN_TRYLOCK(&l->lock);
945 if (unlikely(r))
946 return r;
947 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
948
949 l->info.owner = tid_bit;
950 l->info.last_location.function = func;
951 l->info.last_location.file = file;
952 l->info.last_location.line = line;
953
954 return 0;
955}
956
957static inline void __spin_unlock(enum lock_label lbl, struct ha_spinlock *l,
958 const char *func, const char *file, int line)
959{
960 if (unlikely(!(l->info.owner & tid_bit))) {
961 /* the thread is not owning the lock */
962 abort();
963 }
964
965 l->info.owner = 0;
966 l->info.last_location.function = func;
967 l->info.last_location.file = file;
968 l->info.last_location.line = line;
969
Willy Tarreau7c2a2ad2017-11-02 16:26:02 +0100970 __SPIN_UNLOCK(&l->lock);
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200971 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_unlocked, 1);
972}
973
974#else /* DEBUG_THREAD */
975
976#define HA_SPINLOCK_T unsigned long
977
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100978#define HA_SPIN_INIT(l) ({ (*l) = 0; })
979#define HA_SPIN_DESTROY(l) ({ (*l) = 0; })
980#define HA_SPIN_LOCK(lbl, l) pl_take_s(l)
981#define HA_SPIN_TRYLOCK(lbl, l) !pl_try_s(l)
982#define HA_SPIN_UNLOCK(lbl, l) pl_drop_s(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200983
984#define HA_RWLOCK_T unsigned long
985
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100986#define HA_RWLOCK_INIT(l) ({ (*l) = 0; })
987#define HA_RWLOCK_DESTROY(l) ({ (*l) = 0; })
988#define HA_RWLOCK_WRLOCK(lbl,l) pl_take_w(l)
989#define HA_RWLOCK_TRYWRLOCK(lbl,l) !pl_try_w(l)
990#define HA_RWLOCK_WRUNLOCK(lbl,l) pl_drop_w(l)
991#define HA_RWLOCK_RDLOCK(lbl,l) pl_take_r(l)
992#define HA_RWLOCK_TRYRDLOCK(lbl,l) !pl_try_r(l)
993#define HA_RWLOCK_RDUNLOCK(lbl,l) pl_drop_r(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200994
995#endif /* DEBUG_THREAD */
996
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100997#ifdef __x86_64__
Willy Tarreau2325d8a2018-10-10 18:29:23 +0200998
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100999static __inline int
1000__ha_cas_dw(void *target, void *compare, const void *set)
1001{
1002 char ret;
1003
1004 __asm __volatile("lock cmpxchg16b %0; setz %3"
1005 : "+m" (*(void **)target),
1006 "=a" (((void **)compare)[0]),
1007 "=d" (((void **)compare)[1]),
1008 "=q" (ret)
1009 : "a" (((void **)compare)[0]),
1010 "d" (((void **)compare)[1]),
1011 "b" (((const void **)set)[0]),
1012 "c" (((const void **)set)[1])
1013 : "memory", "cc");
1014 return (ret);
1015}
1016
Olivier Houchard9abcf6e2019-03-07 18:45:00 +01001017/* Use __ha_barrier_atomic* when you're trying to protect data that are
1018 * are modified using HA_ATOMIC* (except HA_ATOMIC_STORE)
1019 */
1020static __inline void
1021__ha_barrier_atomic_load(void)
1022{
1023 __asm __volatile("" ::: "memory");
1024}
1025
Olivier Houchardf61f0cb2017-12-21 17:13:05 +01001026static __inline void
Olivier Houchard9abcf6e2019-03-07 18:45:00 +01001027__ha_barrier_atomic_store(void)
1028{
1029 __asm __volatile("" ::: "memory");
1030}
1031
1032static __inline void
1033__ha_barrier_atomic_full(void)
1034{
1035 __asm __volatile("" ::: "memory");
1036}
1037
1038static __inline void
Olivier Houchardf61f0cb2017-12-21 17:13:05 +01001039__ha_barrier_load(void)
1040{
1041 __asm __volatile("lfence" ::: "memory");
1042}
1043
1044static __inline void
1045__ha_barrier_store(void)
1046{
1047 __asm __volatile("sfence" ::: "memory");
1048}
1049
1050static __inline void
1051__ha_barrier_full(void)
1052{
1053 __asm __volatile("mfence" ::: "memory");
1054}
1055
1056#elif defined(__arm__) && (defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__))
Willy Tarreau2325d8a2018-10-10 18:29:23 +02001057
Olivier Houchard9abcf6e2019-03-07 18:45:00 +01001058/* Use __ha_barrier_atomic* when you're trying to protect data that are
1059 * are modified using HA_ATOMIC* (except HA_ATOMIC_STORE)
1060 */
1061static __inline void
1062__ha_barrier_atomic_load(void)
1063{
1064 __asm __volatile("dmb" ::: "memory");
1065}
1066
1067static __inline void
1068__ha_barrier_atomic_store(void)
1069{
1070 __asm __volatile("dsb" ::: "memory");
1071}
1072
1073static __inline void
1074__ha_barrier_atomic_full(void)
1075{
1076 __asm __volatile("dmb" ::: "memory");
1077}
1078
Olivier Houchardf61f0cb2017-12-21 17:13:05 +01001079static __inline void
1080__ha_barrier_load(void)
1081{
1082 __asm __volatile("dmb" ::: "memory");
1083}
1084
1085static __inline void
1086__ha_barrier_store(void)
1087{
1088 __asm __volatile("dsb" ::: "memory");
1089}
1090
1091static __inline void
1092__ha_barrier_full(void)
1093{
1094 __asm __volatile("dmb" ::: "memory");
1095}
1096
Willy Tarreau41ccb192018-02-14 14:16:28 +01001097static __inline int __ha_cas_dw(void *target, void *compare, const void *set)
Olivier Houchardf61f0cb2017-12-21 17:13:05 +01001098{
1099 uint64_t previous;
1100 int tmp;
1101
1102 __asm __volatile("1:"
1103 "ldrexd %0, [%4];"
1104 "cmp %Q0, %Q2;"
1105 "ittt eq;"
1106 "cmpeq %R0, %R2;"
1107 "strexdeq %1, %3, [%4];"
1108 "cmpeq %1, #1;"
1109 "beq 1b;"
1110 : "=&r" (previous), "=&r" (tmp)
Willy Tarreau41ccb192018-02-14 14:16:28 +01001111 : "r" (*(uint64_t *)compare), "r" (*(uint64_t *)set), "r" (target)
Olivier Houchardf61f0cb2017-12-21 17:13:05 +01001112 : "memory", "cc");
1113 tmp = (previous == *(uint64_t *)compare);
1114 *(uint64_t *)compare = previous;
1115 return (tmp);
1116}
1117
1118#elif defined (__aarch64__)
Olivier Houchardf61f0cb2017-12-21 17:13:05 +01001119
Olivier Houchard9abcf6e2019-03-07 18:45:00 +01001120/* Use __ha_barrier_atomic* when you're trying to protect data that are
1121 * are modified using HA_ATOMIC* (except HA_ATOMIC_STORE)
1122 */
1123static __inline void
1124__ha_barrier_atomic_load(void)
1125{
1126 __asm __volatile("dmb ishld" ::: "memory");
1127}
1128
1129static __inline void
1130__ha_barrier_atomic_store(void)
1131{
1132 __asm __volatile("dmb ishst" ::: "memory");
1133}
1134
1135static __inline void
1136__ha_barrier_atomic_full(void)
1137{
1138 __asm __volatile("dmb ish" ::: "memory");
1139}
1140
Olivier Houchardf61f0cb2017-12-21 17:13:05 +01001141static __inline void
1142__ha_barrier_load(void)
1143{
1144 __asm __volatile("dmb ishld" ::: "memory");
1145}
1146
1147static __inline void
1148__ha_barrier_store(void)
1149{
1150 __asm __volatile("dmb ishst" ::: "memory");
1151}
1152
1153static __inline void
1154__ha_barrier_full(void)
1155{
1156 __asm __volatile("dmb ish" ::: "memory");
1157}
1158
1159static __inline int __ha_cas_dw(void *target, void *compare, void *set)
1160{
1161 void *value[2];
1162 uint64_t tmp1, tmp2;
1163
1164 __asm__ __volatile__("1:"
1165 "ldxp %0, %1, [%4];"
1166 "mov %2, %0;"
1167 "mov %3, %1;"
1168 "eor %0, %0, %5;"
1169 "eor %1, %1, %6;"
1170 "orr %1, %0, %1;"
1171 "mov %w0, #0;"
1172 "cbnz %1, 2f;"
1173 "stxp %w0, %7, %8, [%4];"
1174 "cbnz %w0, 1b;"
1175 "mov %w0, #1;"
1176 "2:"
1177 : "=&r" (tmp1), "=&r" (tmp2), "=&r" (value[0]), "=&r" (value[1])
1178 : "r" (target), "r" (((void **)(compare))[0]), "r" (((void **)(compare))[1]), "r" (((void **)(set))[0]), "r" (((void **)(set))[1])
1179 : "cc", "memory");
1180
1181 memcpy(compare, &value, sizeof(value));
1182 return (tmp1);
1183}
1184
1185#else
Olivier Houchard9abcf6e2019-03-07 18:45:00 +01001186#define __ha_barrier_atomic_load __sync_synchronize
1187#define __ha_barrier_atomic_store __sync_synchronize
1188#define __ha_barrier_atomic_full __sync_synchronize
Olivier Houchardf61f0cb2017-12-21 17:13:05 +01001189#define __ha_barrier_load __sync_synchronize
1190#define __ha_barrier_store __sync_synchronize
1191#define __ha_barrier_full __sync_synchronize
1192#endif
1193
Willy Tarreaua8ae77d2018-11-25 19:28:23 +01001194void ha_spin_init(HA_SPINLOCK_T *l);
1195void ha_rwlock_init(HA_RWLOCK_T *l);
1196
Christopher Faulet1a2b56e2017-10-12 16:09:09 +02001197#endif /* USE_THREAD */
1198
Willy Tarreau149ab772019-01-26 14:27:06 +01001199extern int thread_cpus_enabled_at_boot;
1200
Olivier Houchardf61f0cb2017-12-21 17:13:05 +01001201static inline void __ha_compiler_barrier(void)
1202{
1203 __asm __volatile("" ::: "memory");
1204}
1205
Willy Tarreau0ccd3222018-07-30 10:34:35 +02001206int parse_nbthread(const char *arg, char **err);
Willy Tarreau149ab772019-01-26 14:27:06 +01001207int thread_get_default_count();
Willy Tarreau4037a3f2018-03-28 18:06:47 +02001208
Olivier Houchardd0c3b882019-03-07 18:55:31 +01001209#ifndef _HA_ATOMIC_CAS
1210#define _HA_ATOMIC_CAS HA_ATOMIC_CAS
1211#endif /* !_HA_ATOMIC_CAS */
1212
Willy Tarreau6a38b322019-05-11 18:04:24 +02001213#ifndef _HA_ATOMIC_DWCAS
1214#define _HA_ATOMIC_DWCAS HA_ATOMIC_DWCAS
1215#endif /* !_HA_ATOMIC_CAS */
1216
Olivier Houchardd0c3b882019-03-07 18:55:31 +01001217#ifndef _HA_ATOMIC_ADD
1218#define _HA_ATOMIC_ADD HA_ATOMIC_ADD
1219#endif /* !_HA_ATOMIC_ADD */
1220
1221#ifndef _HA_ATOMIC_XADD
1222#define _HA_ATOMIC_XADD HA_ATOMIC_XADD
1223#endif /* !_HA_ATOMIC_SUB */
1224
1225#ifndef _HA_ATOMIC_SUB
1226#define _HA_ATOMIC_SUB HA_ATOMIC_SUB
1227#endif /* !_HA_ATOMIC_SUB */
1228
1229#ifndef _HA_ATOMIC_AND
1230#define _HA_ATOMIC_AND HA_ATOMIC_AND
1231#endif /* !_HA_ATOMIC_AND */
1232
1233#ifndef _HA_ATOMIC_OR
1234#define _HA_ATOMIC_OR HA_ATOMIC_OR
1235#endif /* !_HA_ATOMIC_OR */
1236
1237#ifndef _HA_ATOMIC_XCHG
1238#define _HA_ATOMIC_XCHG HA_ATOMIC_XCHG
1239#endif /* !_HA_ATOMIC_XCHG */
1240
1241#ifndef _HA_ATOMIC_STORE
1242#define _HA_ATOMIC_STORE HA_ATOMIC_STORE
1243#endif /* !_HA_ATOMIC_STORE */
Olivier Houchard9ce62b52019-04-30 13:38:02 +02001244
1245#ifndef _HA_ATOMIC_LOAD
1246#define _HA_ATOMIC_LOAD HA_ATOMIC_LOAD
1247#endif /* !_HA_ATOMIC_LOAD */
Christopher Faulet1a2b56e2017-10-12 16:09:09 +02001248#endif /* _COMMON_HATHREADS_H */