blob: 0cd00fb00f632cafc548341aac3d472f45cc0da6 [file] [log] [blame]
Christopher Faulet1a2b56e2017-10-12 16:09:09 +02001/*
2 * include/common/hathreads.h
3 * definitions, macros and inline functions about threads.
4 *
5 * Copyright (C) 2017 Christopher Fauet - cfaulet@haproxy.com
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation, version 2.1
10 * exclusively.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22#ifndef _COMMON_HATHREADS_H
23#define _COMMON_HATHREADS_H
24
25#include <common/config.h>
Willy Tarreau90fa97b2018-11-25 19:46:08 +010026#include <common/initcall.h>
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020027
Willy Tarreau0ccd3222018-07-30 10:34:35 +020028/* Note about all_threads_mask :
Willy Tarreauda9e9392019-02-02 17:03:41 +010029 * - this variable is comprised between 1 and LONGBITS.
30 * - with threads support disabled, this symbol is defined as constant 1UL.
31 * - with threads enabled, it contains the mask of enabled threads. Thus if
32 * only one thread is enabled, it equals 1.
Willy Tarreau0ccd3222018-07-30 10:34:35 +020033 */
34
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020035#ifndef USE_THREAD
36
Willy Tarreau421f02e2018-01-20 18:19:22 +010037#define MAX_THREADS 1
Willy Tarreau0c026f42018-08-01 19:12:20 +020038#define MAX_THREADS_MASK 1
39
40/* Only way found to replace variables with constants that are optimized away
41 * at build time.
42 */
43enum { all_threads_mask = 1UL };
44enum { tid_bit = 1UL };
45enum { tid = 0 };
Willy Tarreau421f02e2018-01-20 18:19:22 +010046
Christopher Faulet9dcf9b62017-11-13 10:34:01 +010047#define __decl_hathreads(decl)
Willy Tarreau90fa97b2018-11-25 19:46:08 +010048#define __decl_spinlock(lock)
49#define __decl_aligned_spinlock(lock)
50#define __decl_rwlock(lock)
51#define __decl_aligned_rwlock(lock)
Christopher Faulet9dcf9b62017-11-13 10:34:01 +010052
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020053#define HA_ATOMIC_CAS(val, old, new) ({((*val) == (*old)) ? (*(val) = (new) , 1) : (*(old) = *(val), 0);})
54#define HA_ATOMIC_ADD(val, i) ({*(val) += (i);})
55#define HA_ATOMIC_SUB(val, i) ({*(val) -= (i);})
Willy Tarreau9378df82018-09-05 16:11:03 +020056#define HA_ATOMIC_XADD(val, i) \
57 ({ \
58 typeof((val)) __p_xadd = (val); \
59 typeof(*(val)) __old_xadd = *__p_xadd; \
60 *__p_xadd += i; \
61 __old_xadd; \
62 })
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020063#define HA_ATOMIC_AND(val, flags) ({*(val) &= (flags);})
64#define HA_ATOMIC_OR(val, flags) ({*(val) |= (flags);})
65#define HA_ATOMIC_XCHG(val, new) \
66 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020067 typeof(*(val)) __old_xchg = *(val); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020068 *(val) = new; \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020069 __old_xchg; \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020070 })
Willy Tarreau5266b3e2018-01-25 17:43:58 +010071#define HA_ATOMIC_BTS(val, bit) \
72 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020073 typeof((val)) __p_bts = (val); \
74 typeof(*__p_bts) __b_bts = (1UL << (bit)); \
75 typeof(*__p_bts) __t_bts = *__p_bts & __b_bts; \
76 if (!__t_bts) \
77 *__p_bts |= __b_bts; \
78 __t_bts; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +010079 })
80#define HA_ATOMIC_BTR(val, bit) \
81 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020082 typeof((val)) __p_btr = (val); \
83 typeof(*__p_btr) __b_btr = (1UL << (bit)); \
84 typeof(*__p_btr) __t_btr = *__p_btr & __b_btr; \
85 if (__t_btr) \
86 *__p_btr &= ~__b_btr; \
87 __t_btr; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +010088 })
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020089#define HA_ATOMIC_STORE(val, new) ({*(val) = new;})
90#define HA_ATOMIC_UPDATE_MAX(val, new) \
91 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020092 typeof(*(val)) __new_max = (new); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020093 \
Christopher Faulet48aa13f2018-04-09 08:45:43 +020094 if (*(val) < __new_max) \
95 *(val) = __new_max; \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020096 *(val); \
97 })
98
99#define HA_ATOMIC_UPDATE_MIN(val, new) \
100 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200101 typeof(*(val)) __new_min = (new); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200102 \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200103 if (*(val) > __new_min) \
104 *(val) = __new_min; \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200105 *(val); \
106 })
107
Willy Tarreaub29dc952017-10-31 18:00:20 +0100108#define HA_BARRIER() do { } while (0)
Christopher Faulet339fff82017-10-19 11:59:15 +0200109
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100110#define HA_SPIN_INIT(l) do { /* do nothing */ } while(0)
111#define HA_SPIN_DESTROY(l) do { /* do nothing */ } while(0)
112#define HA_SPIN_LOCK(lbl, l) do { /* do nothing */ } while(0)
113#define HA_SPIN_TRYLOCK(lbl, l) ({ 0; })
114#define HA_SPIN_UNLOCK(lbl, l) do { /* do nothing */ } while(0)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200115
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100116#define HA_RWLOCK_INIT(l) do { /* do nothing */ } while(0)
117#define HA_RWLOCK_DESTROY(l) do { /* do nothing */ } while(0)
118#define HA_RWLOCK_WRLOCK(lbl, l) do { /* do nothing */ } while(0)
119#define HA_RWLOCK_TRYWRLOCK(lbl, l) ({ 0; })
120#define HA_RWLOCK_WRUNLOCK(lbl, l) do { /* do nothing */ } while(0)
121#define HA_RWLOCK_RDLOCK(lbl, l) do { /* do nothing */ } while(0)
122#define HA_RWLOCK_TRYRDLOCK(lbl, l) ({ 0; })
123#define HA_RWLOCK_RDUNLOCK(lbl, l) do { /* do nothing */ } while(0)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200124
William Lallemand6e1796e2018-06-07 11:23:40 +0200125#define ha_sigmask(how, set, oldset) sigprocmask(how, set, oldset)
126
Willy Tarreau0c026f42018-08-01 19:12:20 +0200127static inline void ha_set_tid(unsigned int tid)
128{
129}
William Lallemand6e1796e2018-06-07 11:23:40 +0200130
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100131static inline void __ha_barrier_load(void)
132{
133}
134
135static inline void __ha_barrier_store(void)
136{
137}
138
139static inline void __ha_barrier_full(void)
140{
141}
142
Willy Tarreau60b639c2018-08-02 10:16:17 +0200143static inline void thread_harmless_now()
144{
145}
146
147static inline void thread_harmless_end()
148{
149}
150
151static inline void thread_isolate()
152{
153}
154
155static inline void thread_release()
156{
157}
158
159static inline unsigned long thread_isolated()
160{
161 return 1;
162}
163
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200164#else /* USE_THREAD */
165
166#include <stdio.h>
167#include <stdlib.h>
168#include <string.h>
169#include <pthread.h>
170#include <import/plock.h>
171
Willy Tarreauf5809cd2019-01-26 13:35:03 +0100172#ifndef MAX_THREADS
Willy Tarreau421f02e2018-01-20 18:19:22 +0100173#define MAX_THREADS LONGBITS
Willy Tarreauf5809cd2019-01-26 13:35:03 +0100174#endif
175
176#define MAX_THREADS_MASK (~0UL >> (LONGBITS - MAX_THREADS))
Willy Tarreau421f02e2018-01-20 18:19:22 +0100177
Christopher Faulet9dcf9b62017-11-13 10:34:01 +0100178#define __decl_hathreads(decl) decl
179
Willy Tarreau90fa97b2018-11-25 19:46:08 +0100180/* declare a self-initializing spinlock */
181#define __decl_spinlock(lock) \
182 HA_SPINLOCK_T (lock); \
183 INITCALL1(STG_LOCK, ha_spin_init, &(lock))
184
185/* declare a self-initializing spinlock, aligned on a cache line */
186#define __decl_aligned_spinlock(lock) \
187 HA_SPINLOCK_T (lock) __attribute__((aligned(64))); \
188 INITCALL1(STG_LOCK, ha_spin_init, &(lock))
189
190/* declare a self-initializing rwlock */
191#define __decl_rwlock(lock) \
192 HA_RWLOCK_T (lock); \
193 INITCALL1(STG_LOCK, ha_rwlock_init, &(lock))
194
195/* declare a self-initializing rwlock, aligned on a cache line */
196#define __decl_aligned_rwlock(lock) \
197 HA_RWLOCK_T (lock) __attribute__((aligned(64))); \
198 INITCALL1(STG_LOCK, ha_rwlock_init, &(lock))
199
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200200/* TODO: thread: For now, we rely on GCC builtins but it could be a good idea to
201 * have a header file regrouping all functions dealing with threads. */
Willy Tarreau1a69af62018-01-04 18:49:31 +0100202
David Carlierec5e8452018-01-11 14:20:43 +0000203#if defined(__GNUC__) && (__GNUC__ < 4 || __GNUC__ == 4 && __GNUC_MINOR__ < 7) && !defined(__clang__)
Willy Tarreau1a69af62018-01-04 18:49:31 +0100204/* gcc < 4.7 */
205
206#define HA_ATOMIC_ADD(val, i) __sync_add_and_fetch(val, i)
207#define HA_ATOMIC_SUB(val, i) __sync_sub_and_fetch(val, i)
Willy Tarreau9378df82018-09-05 16:11:03 +0200208#define HA_ATOMIC_XADD(val, i) __sync_fetch_and_add(val, i)
Willy Tarreau1a69af62018-01-04 18:49:31 +0100209#define HA_ATOMIC_AND(val, flags) __sync_and_and_fetch(val, flags)
210#define HA_ATOMIC_OR(val, flags) __sync_or_and_fetch(val, flags)
211
212/* the CAS is a bit complicated. The older API doesn't support returning the
213 * value and the swap's result at the same time. So here we take what looks
214 * like the safest route, consisting in using the boolean version guaranteeing
215 * that the operation was performed or not, and we snoop a previous value. If
216 * the compare succeeds, we return. If it fails, we return the previous value,
217 * but only if it differs from the expected one. If it's the same it's a race
218 * thus we try again to avoid confusing a possibly sensitive caller.
219 */
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200220#define HA_ATOMIC_CAS(val, old, new) \
221 ({ \
222 typeof((val)) __val_cas = (val); \
223 typeof((old)) __oldp_cas = (old); \
224 typeof(*(old)) __oldv_cas; \
225 typeof((new)) __new_cas = (new); \
226 int __ret_cas; \
227 do { \
228 __oldv_cas = *__val_cas; \
229 __ret_cas = __sync_bool_compare_and_swap(__val_cas, *__oldp_cas, __new_cas); \
230 } while (!__ret_cas && *__oldp_cas == __oldv_cas); \
231 if (!__ret_cas) \
232 *__oldp_cas = __oldv_cas; \
233 __ret_cas; \
Willy Tarreau1a69af62018-01-04 18:49:31 +0100234 })
235
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200236#define HA_ATOMIC_XCHG(val, new) \
237 ({ \
238 typeof((val)) __val_xchg = (val); \
239 typeof(*(val)) __old_xchg; \
240 typeof((new)) __new_xchg = (new); \
241 do { __old_xchg = *__val_xchg; \
242 } while (!__sync_bool_compare_and_swap(__val_xchg, __old_xchg, __new_xchg)); \
243 __old_xchg; \
Willy Tarreau1a69af62018-01-04 18:49:31 +0100244 })
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100245
246#define HA_ATOMIC_BTS(val, bit) \
247 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200248 typeof(*(val)) __b_bts = (1UL << (bit)); \
249 __sync_fetch_and_or((val), __b_bts) & __b_bts; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100250 })
251
252#define HA_ATOMIC_BTR(val, bit) \
253 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200254 typeof(*(val)) __b_btr = (1UL << (bit)); \
255 __sync_fetch_and_and((val), ~__b_btr) & __b_btr; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100256 })
257
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200258#define HA_ATOMIC_STORE(val, new) \
259 ({ \
260 typeof((val)) __val_store = (val); \
261 typeof(*(val)) __old_store; \
262 typeof((new)) __new_store = (new); \
263 do { __old_store = *__val_store; \
264 } while (!__sync_bool_compare_and_swap(__val_store, __old_store, __new_store)); \
Willy Tarreau1a69af62018-01-04 18:49:31 +0100265 })
266#else
267/* gcc >= 4.7 */
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200268#define HA_ATOMIC_CAS(val, old, new) __atomic_compare_exchange_n(val, old, new, 0, 0, 0)
269#define HA_ATOMIC_ADD(val, i) __atomic_add_fetch(val, i, 0)
Willy Tarreau9378df82018-09-05 16:11:03 +0200270#define HA_ATOMIC_XADD(val, i) __atomic_fetch_add(val, i, 0)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200271#define HA_ATOMIC_SUB(val, i) __atomic_sub_fetch(val, i, 0)
272#define HA_ATOMIC_AND(val, flags) __atomic_and_fetch(val, flags, 0)
273#define HA_ATOMIC_OR(val, flags) __atomic_or_fetch(val, flags, 0)
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100274#define HA_ATOMIC_BTS(val, bit) \
275 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200276 typeof(*(val)) __b_bts = (1UL << (bit)); \
277 __sync_fetch_and_or((val), __b_bts) & __b_bts; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100278 })
279
280#define HA_ATOMIC_BTR(val, bit) \
281 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200282 typeof(*(val)) __b_btr = (1UL << (bit)); \
283 __sync_fetch_and_and((val), ~__b_btr) & __b_btr; \
Willy Tarreau5266b3e2018-01-25 17:43:58 +0100284 })
285
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200286#define HA_ATOMIC_XCHG(val, new) __atomic_exchange_n(val, new, 0)
287#define HA_ATOMIC_STORE(val, new) __atomic_store_n(val, new, 0)
Willy Tarreau1a69af62018-01-04 18:49:31 +0100288#endif
289
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200290#define HA_ATOMIC_UPDATE_MAX(val, new) \
291 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200292 typeof(*(val)) __old_max = *(val); \
293 typeof(*(val)) __new_max = (new); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200294 \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200295 while (__old_max < __new_max && \
296 !HA_ATOMIC_CAS(val, &__old_max, __new_max)); \
297 *(val); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200298 })
299#define HA_ATOMIC_UPDATE_MIN(val, new) \
300 ({ \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200301 typeof(*(val)) __old_min = *(val); \
302 typeof(*(val)) __new_min = (new); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200303 \
Christopher Faulet48aa13f2018-04-09 08:45:43 +0200304 while (__old_min > __new_min && \
305 !HA_ATOMIC_CAS(val, &__old_min, __new_min)); \
306 *(val); \
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200307 })
308
Willy Tarreaub29dc952017-10-31 18:00:20 +0100309#define HA_BARRIER() pl_barrier()
310
Willy Tarreau60b639c2018-08-02 10:16:17 +0200311void thread_harmless_till_end();
312void thread_isolate();
313void thread_release();
Christopher Faulet339fff82017-10-19 11:59:15 +0200314
Willy Tarreau0c026f42018-08-01 19:12:20 +0200315extern THREAD_LOCAL unsigned int tid; /* The thread id */
316extern THREAD_LOCAL unsigned long tid_bit; /* The bit corresponding to the thread id */
Christopher Fauletddb6c162018-07-20 09:31:53 +0200317extern volatile unsigned long all_threads_mask;
Willy Tarreau60b639c2018-08-02 10:16:17 +0200318extern volatile unsigned long threads_want_rdv_mask;
319extern volatile unsigned long threads_harmless_mask;
320
321/* explanation for threads_want_rdv_mask and threads_harmless_mask :
322 * - threads_want_rdv_mask is a bit field indicating all threads that have
323 * requested a rendez-vous of other threads using thread_isolate().
324 * - threads_harmless_mask is a bit field indicating all threads that are
325 * currently harmless in that they promise not to access a shared resource.
326 *
327 * For a given thread, its bits in want_rdv and harmless can be translated like
328 * this :
329 *
330 * ----------+----------+----------------------------------------------------
331 * want_rdv | harmless | description
332 * ----------+----------+----------------------------------------------------
333 * 0 | 0 | thread not interested in RDV, possibly harmful
334 * 0 | 1 | thread not interested in RDV but harmless
335 * 1 | 1 | thread interested in RDV and waiting for its turn
336 * 1 | 0 | thread currently working isolated from others
337 * ----------+----------+----------------------------------------------------
338 */
Olivier Houchard6b96f722018-04-25 16:58:25 +0200339
William Lallemand6e1796e2018-06-07 11:23:40 +0200340#define ha_sigmask(how, set, oldset) pthread_sigmask(how, set, oldset)
341
Willy Tarreau0c026f42018-08-01 19:12:20 +0200342/* sets the thread ID and the TID bit for the current thread */
343static inline void ha_set_tid(unsigned int data)
344{
345 tid = data;
346 tid_bit = (1UL << tid);
347}
348
Willy Tarreau60b639c2018-08-02 10:16:17 +0200349/* Marks the thread as harmless. Note: this must be true, i.e. the thread must
350 * not be touching any unprotected shared resource during this period. Usually
351 * this is called before poll(), but it may also be placed around very slow
352 * calls (eg: some crypto operations). Needs to be terminated using
353 * thread_harmless_end().
354 */
355static inline void thread_harmless_now()
356{
357 HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
358}
359
360/* Ends the harmless period started by thread_harmless_now(). Usually this is
361 * placed after the poll() call. If it is discovered that a job was running and
362 * is relying on the thread still being harmless, the thread waits for the
363 * other one to finish.
364 */
365static inline void thread_harmless_end()
366{
367 while (1) {
368 HA_ATOMIC_AND(&threads_harmless_mask, ~tid_bit);
369 if (likely((threads_want_rdv_mask & all_threads_mask) == 0))
370 break;
371 thread_harmless_till_end();
372 }
373}
374
375/* an isolated thread has harmless cleared and want_rdv set */
376static inline unsigned long thread_isolated()
377{
378 return threads_want_rdv_mask & ~threads_harmless_mask & tid_bit;
379}
380
William Lallemand6e1796e2018-06-07 11:23:40 +0200381
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200382#if defined(DEBUG_THREAD) || defined(DEBUG_FULL)
383
Christopher Fauletf51bac22018-01-30 11:04:29 +0100384/* WARNING!!! if you update this enum, please also keep lock_label() up to date below */
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200385enum lock_label {
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200386 FD_LOCK,
Emeric Brunc60def82017-09-27 14:59:38 +0200387 TASK_RQ_LOCK,
388 TASK_WQ_LOCK,
Christopher Fauletb349e482017-08-29 09:52:38 +0200389 POOL_LOCK,
Christopher Faulet8d8aa0d2017-05-30 15:36:50 +0200390 LISTENER_LOCK,
391 LISTENER_QUEUE_LOCK,
Christopher Fauletff8abcd2017-06-02 15:33:24 +0200392 PROXY_LOCK,
Christopher Faulet29f77e82017-06-08 14:04:45 +0200393 SERVER_LOCK,
Christopher Faulet5b517552017-06-09 14:17:53 +0200394 LBPRM_LOCK,
Christopher Fauletb79a94c2017-05-30 15:34:30 +0200395 SIGNALS_LOCK,
Emeric Brun819fc6f2017-06-13 19:37:32 +0200396 STK_TABLE_LOCK,
397 STK_SESS_LOCK,
Emeric Brun1138fd02017-06-19 12:38:55 +0200398 APPLETS_LOCK,
Emeric Brun80527f52017-06-19 17:46:37 +0200399 PEER_LOCK,
Emeric Bruna1dd2432017-06-21 15:42:52 +0200400 BUF_WQ_LOCK,
Emeric Brun6b35e9b2017-06-30 16:23:45 +0200401 STRMS_LOCK,
Emeric Brun821bb9b2017-06-15 16:37:39 +0200402 SSL_LOCK,
403 SSL_GEN_CERTS_LOCK,
Emeric Brunb5997f72017-07-03 11:34:05 +0200404 PATREF_LOCK,
405 PATEXP_LOCK,
406 PATLRU_LOCK,
Christopher Faulete95f2c32017-07-24 16:30:34 +0200407 VARS_LOCK,
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +0200408 COMP_POOL_LOCK,
Thierry FOURNIER61ba0e22017-07-12 11:41:21 +0200409 LUA_LOCK,
Thierry FOURNIER738a6d72017-07-17 00:14:07 +0200410 NOTIF_LOCK,
Christopher Faulet24289f22017-09-25 14:48:02 +0200411 SPOE_APPLET_LOCK,
Christopher Fauletb2812a62017-10-04 16:17:58 +0200412 DNS_LOCK,
Christopher Fauletcfda8472017-10-20 15:40:23 +0200413 PID_LIST_LOCK,
Christopher Fauletc2a89a62017-10-23 15:54:24 +0200414 EMAIL_ALERTS_LOCK,
Emeric Brund8b3b652017-11-07 11:19:48 +0100415 PIPES_LOCK,
Willy Tarreau1605c7a2018-01-23 19:01:49 +0100416 START_LOCK,
Christopher Faulet16f45c82018-02-16 11:23:49 +0100417 TLSKEYS_REF_LOCK,
Willy Tarreau34d4b522018-10-29 18:02:54 +0100418 AUTH_LOCK,
Ben51Degrees4ddf59d2019-02-05 13:24:00 +0000419 OTHER_LOCK,
Christopher Faulet339fff82017-10-19 11:59:15 +0200420 LOCK_LABELS
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200421};
422struct lock_stat {
423 uint64_t nsec_wait_for_write;
424 uint64_t nsec_wait_for_read;
425 uint64_t num_write_locked;
426 uint64_t num_write_unlocked;
427 uint64_t num_read_locked;
428 uint64_t num_read_unlocked;
429};
430
431extern struct lock_stat lock_stats[LOCK_LABELS];
432
433#define __HA_SPINLOCK_T unsigned long
434
435#define __SPIN_INIT(l) ({ (*l) = 0; })
436#define __SPIN_DESTROY(l) ({ (*l) = 0; })
Willy Tarreau88ac59b2017-11-06 01:03:26 +0100437#define __SPIN_LOCK(l) pl_take_s(l)
438#define __SPIN_TRYLOCK(l) !pl_try_s(l)
439#define __SPIN_UNLOCK(l) pl_drop_s(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200440
441#define __HA_RWLOCK_T unsigned long
442
443#define __RWLOCK_INIT(l) ({ (*l) = 0; })
444#define __RWLOCK_DESTROY(l) ({ (*l) = 0; })
445#define __RWLOCK_WRLOCK(l) pl_take_w(l)
446#define __RWLOCK_TRYWRLOCK(l) !pl_try_w(l)
447#define __RWLOCK_WRUNLOCK(l) pl_drop_w(l)
448#define __RWLOCK_RDLOCK(l) pl_take_r(l)
449#define __RWLOCK_TRYRDLOCK(l) !pl_try_r(l)
450#define __RWLOCK_RDUNLOCK(l) pl_drop_r(l)
451
452#define HA_SPINLOCK_T struct ha_spinlock
453
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100454#define HA_SPIN_INIT(l) __spin_init(l)
455#define HA_SPIN_DESTROY(l) __spin_destroy(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200456
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100457#define HA_SPIN_LOCK(lbl, l) __spin_lock(lbl, l, __func__, __FILE__, __LINE__)
458#define HA_SPIN_TRYLOCK(lbl, l) __spin_trylock(lbl, l, __func__, __FILE__, __LINE__)
459#define HA_SPIN_UNLOCK(lbl, l) __spin_unlock(lbl, l, __func__, __FILE__, __LINE__)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200460
461#define HA_RWLOCK_T struct ha_rwlock
462
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100463#define HA_RWLOCK_INIT(l) __ha_rwlock_init((l))
464#define HA_RWLOCK_DESTROY(l) __ha_rwlock_destroy((l))
465#define HA_RWLOCK_WRLOCK(lbl,l) __ha_rwlock_wrlock(lbl, l, __func__, __FILE__, __LINE__)
466#define HA_RWLOCK_TRYWRLOCK(lbl,l) __ha_rwlock_trywrlock(lbl, l, __func__, __FILE__, __LINE__)
467#define HA_RWLOCK_WRUNLOCK(lbl,l) __ha_rwlock_wrunlock(lbl, l, __func__, __FILE__, __LINE__)
468#define HA_RWLOCK_RDLOCK(lbl,l) __ha_rwlock_rdlock(lbl, l)
469#define HA_RWLOCK_TRYRDLOCK(lbl,l) __ha_rwlock_tryrdlock(lbl, l)
470#define HA_RWLOCK_RDUNLOCK(lbl,l) __ha_rwlock_rdunlock(lbl, l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200471
472struct ha_spinlock {
473 __HA_SPINLOCK_T lock;
474 struct {
475 unsigned long owner; /* a bit is set to 1 << tid for the lock owner */
476 unsigned long waiters; /* a bit is set to 1 << tid for waiting threads */
477 struct {
478 const char *function;
479 const char *file;
480 int line;
481 } last_location; /* location of the last owner */
482 } info;
483};
484
485struct ha_rwlock {
486 __HA_RWLOCK_T lock;
487 struct {
488 unsigned long cur_writer; /* a bit is set to 1 << tid for the lock owner */
489 unsigned long wait_writers; /* a bit is set to 1 << tid for waiting writers */
490 unsigned long cur_readers; /* a bit is set to 1 << tid for current readers */
491 unsigned long wait_readers; /* a bit is set to 1 << tid for waiting waiters */
492 struct {
493 const char *function;
494 const char *file;
495 int line;
496 } last_location; /* location of the last write owner */
497 } info;
498};
499
Christopher Fauletf51bac22018-01-30 11:04:29 +0100500static inline const char *lock_label(enum lock_label label)
501{
502 switch (label) {
Christopher Fauletf51bac22018-01-30 11:04:29 +0100503 case FD_LOCK: return "FD";
504 case TASK_RQ_LOCK: return "TASK_RQ";
505 case TASK_WQ_LOCK: return "TASK_WQ";
506 case POOL_LOCK: return "POOL";
507 case LISTENER_LOCK: return "LISTENER";
508 case LISTENER_QUEUE_LOCK: return "LISTENER_QUEUE";
509 case PROXY_LOCK: return "PROXY";
510 case SERVER_LOCK: return "SERVER";
Christopher Fauletf51bac22018-01-30 11:04:29 +0100511 case LBPRM_LOCK: return "LBPRM";
512 case SIGNALS_LOCK: return "SIGNALS";
513 case STK_TABLE_LOCK: return "STK_TABLE";
514 case STK_SESS_LOCK: return "STK_SESS";
515 case APPLETS_LOCK: return "APPLETS";
516 case PEER_LOCK: return "PEER";
517 case BUF_WQ_LOCK: return "BUF_WQ";
518 case STRMS_LOCK: return "STRMS";
519 case SSL_LOCK: return "SSL";
520 case SSL_GEN_CERTS_LOCK: return "SSL_GEN_CERTS";
521 case PATREF_LOCK: return "PATREF";
522 case PATEXP_LOCK: return "PATEXP";
523 case PATLRU_LOCK: return "PATLRU";
524 case VARS_LOCK: return "VARS";
525 case COMP_POOL_LOCK: return "COMP_POOL";
526 case LUA_LOCK: return "LUA";
527 case NOTIF_LOCK: return "NOTIF";
528 case SPOE_APPLET_LOCK: return "SPOE_APPLET";
529 case DNS_LOCK: return "DNS";
530 case PID_LIST_LOCK: return "PID_LIST";
531 case EMAIL_ALERTS_LOCK: return "EMAIL_ALERTS";
532 case PIPES_LOCK: return "PIPES";
533 case START_LOCK: return "START";
Christopher Faulet16f45c82018-02-16 11:23:49 +0100534 case TLSKEYS_REF_LOCK: return "TLSKEYS_REF";
Willy Tarreau34d4b522018-10-29 18:02:54 +0100535 case AUTH_LOCK: return "AUTH";
Ben51Degrees4ddf59d2019-02-05 13:24:00 +0000536 case OTHER_LOCK: return "OTHER";
Christopher Fauletf51bac22018-01-30 11:04:29 +0100537 case LOCK_LABELS: break; /* keep compiler happy */
538 };
539 /* only way to come here is consecutive to an internal bug */
540 abort();
541}
542
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200543static inline void show_lock_stats()
544{
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200545 int lbl;
546
547 for (lbl = 0; lbl < LOCK_LABELS; lbl++) {
548 fprintf(stderr,
549 "Stats about Lock %s: \n"
550 "\t # write lock : %lu\n"
551 "\t # write unlock: %lu (%ld)\n"
552 "\t # wait time for write : %.3f msec\n"
553 "\t # wait time for write/lock: %.3f nsec\n"
554 "\t # read lock : %lu\n"
555 "\t # read unlock : %lu (%ld)\n"
556 "\t # wait time for read : %.3f msec\n"
557 "\t # wait time for read/lock : %.3f nsec\n",
Christopher Fauletf51bac22018-01-30 11:04:29 +0100558 lock_label(lbl),
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200559 lock_stats[lbl].num_write_locked,
560 lock_stats[lbl].num_write_unlocked,
561 lock_stats[lbl].num_write_unlocked - lock_stats[lbl].num_write_locked,
562 (double)lock_stats[lbl].nsec_wait_for_write / 1000000.0,
563 lock_stats[lbl].num_write_locked ? ((double)lock_stats[lbl].nsec_wait_for_write / (double)lock_stats[lbl].num_write_locked) : 0,
564 lock_stats[lbl].num_read_locked,
565 lock_stats[lbl].num_read_unlocked,
566 lock_stats[lbl].num_read_unlocked - lock_stats[lbl].num_read_locked,
567 (double)lock_stats[lbl].nsec_wait_for_read / 1000000.0,
568 lock_stats[lbl].num_read_locked ? ((double)lock_stats[lbl].nsec_wait_for_read / (double)lock_stats[lbl].num_read_locked) : 0);
569 }
570}
571
572/* Following functions are used to collect some stats about locks. We wrap
573 * pthread functions to known how much time we wait in a lock. */
574
575static uint64_t nsec_now(void) {
576 struct timespec ts;
577
578 clock_gettime(CLOCK_MONOTONIC, &ts);
579 return ((uint64_t) ts.tv_sec * 1000000000ULL +
580 (uint64_t) ts.tv_nsec);
581}
582
583static inline void __ha_rwlock_init(struct ha_rwlock *l)
584{
585 memset(l, 0, sizeof(struct ha_rwlock));
586 __RWLOCK_INIT(&l->lock);
587}
588
589static inline void __ha_rwlock_destroy(struct ha_rwlock *l)
590{
591 __RWLOCK_DESTROY(&l->lock);
592 memset(l, 0, sizeof(struct ha_rwlock));
593}
594
595
596static inline void __ha_rwlock_wrlock(enum lock_label lbl, struct ha_rwlock *l,
597 const char *func, const char *file, int line)
598{
599 uint64_t start_time;
600
601 if (unlikely(l->info.cur_writer & tid_bit)) {
602 /* the thread is already owning the lock for write */
603 abort();
604 }
605
606 if (unlikely(l->info.cur_readers & tid_bit)) {
607 /* the thread is already owning the lock for read */
608 abort();
609 }
610
611 HA_ATOMIC_OR(&l->info.wait_writers, tid_bit);
612
613 start_time = nsec_now();
614 __RWLOCK_WRLOCK(&l->lock);
615 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_write, (nsec_now() - start_time));
616
617 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
618
619 l->info.cur_writer = tid_bit;
620 l->info.last_location.function = func;
621 l->info.last_location.file = file;
622 l->info.last_location.line = line;
623
624 HA_ATOMIC_AND(&l->info.wait_writers, ~tid_bit);
625}
626
627static inline int __ha_rwlock_trywrlock(enum lock_label lbl, struct ha_rwlock *l,
628 const char *func, const char *file, int line)
629{
630 uint64_t start_time;
631 int r;
632
633 if (unlikely(l->info.cur_writer & tid_bit)) {
634 /* the thread is already owning the lock for write */
635 abort();
636 }
637
638 if (unlikely(l->info.cur_readers & tid_bit)) {
639 /* the thread is already owning the lock for read */
640 abort();
641 }
642
643 /* We set waiting writer because trywrlock could wait for readers to quit */
644 HA_ATOMIC_OR(&l->info.wait_writers, tid_bit);
645
646 start_time = nsec_now();
647 r = __RWLOCK_TRYWRLOCK(&l->lock);
648 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_write, (nsec_now() - start_time));
649 if (unlikely(r)) {
650 HA_ATOMIC_AND(&l->info.wait_writers, ~tid_bit);
651 return r;
652 }
653 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
654
655 l->info.cur_writer = tid_bit;
656 l->info.last_location.function = func;
657 l->info.last_location.file = file;
658 l->info.last_location.line = line;
659
660 HA_ATOMIC_AND(&l->info.wait_writers, ~tid_bit);
661
662 return 0;
663}
664
665static inline void __ha_rwlock_wrunlock(enum lock_label lbl,struct ha_rwlock *l,
666 const char *func, const char *file, int line)
667{
668 if (unlikely(!(l->info.cur_writer & tid_bit))) {
669 /* the thread is not owning the lock for write */
670 abort();
671 }
672
673 l->info.cur_writer = 0;
674 l->info.last_location.function = func;
675 l->info.last_location.file = file;
676 l->info.last_location.line = line;
677
678 __RWLOCK_WRUNLOCK(&l->lock);
679
680 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_unlocked, 1);
681}
682
683static inline void __ha_rwlock_rdlock(enum lock_label lbl,struct ha_rwlock *l)
684{
685 uint64_t start_time;
686
687 if (unlikely(l->info.cur_writer & tid_bit)) {
688 /* the thread is already owning the lock for write */
689 abort();
690 }
691
692 if (unlikely(l->info.cur_readers & tid_bit)) {
693 /* the thread is already owning the lock for read */
694 abort();
695 }
696
697 HA_ATOMIC_OR(&l->info.wait_readers, tid_bit);
698
699 start_time = nsec_now();
700 __RWLOCK_RDLOCK(&l->lock);
701 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_read, (nsec_now() - start_time));
702 HA_ATOMIC_ADD(&lock_stats[lbl].num_read_locked, 1);
703
704 HA_ATOMIC_OR(&l->info.cur_readers, tid_bit);
705
706 HA_ATOMIC_AND(&l->info.wait_readers, ~tid_bit);
707}
708
709static inline int __ha_rwlock_tryrdlock(enum lock_label lbl,struct ha_rwlock *l)
710{
711 int r;
712
713 if (unlikely(l->info.cur_writer & tid_bit)) {
714 /* the thread is already owning the lock for write */
715 abort();
716 }
717
718 if (unlikely(l->info.cur_readers & tid_bit)) {
719 /* the thread is already owning the lock for read */
720 abort();
721 }
722
723 /* try read should never wait */
724 r = __RWLOCK_TRYRDLOCK(&l->lock);
725 if (unlikely(r))
726 return r;
727 HA_ATOMIC_ADD(&lock_stats[lbl].num_read_locked, 1);
728
729 HA_ATOMIC_OR(&l->info.cur_readers, tid_bit);
730
731 return 0;
732}
733
734static inline void __ha_rwlock_rdunlock(enum lock_label lbl,struct ha_rwlock *l)
735{
736 if (unlikely(!(l->info.cur_readers & tid_bit))) {
737 /* the thread is not owning the lock for read */
738 abort();
739 }
740
741 HA_ATOMIC_AND(&l->info.cur_readers, ~tid_bit);
742
743 __RWLOCK_RDUNLOCK(&l->lock);
744
745 HA_ATOMIC_ADD(&lock_stats[lbl].num_read_unlocked, 1);
746}
747
748static inline void __spin_init(struct ha_spinlock *l)
749{
750 memset(l, 0, sizeof(struct ha_spinlock));
751 __SPIN_INIT(&l->lock);
752}
753
754static inline void __spin_destroy(struct ha_spinlock *l)
755{
756 __SPIN_DESTROY(&l->lock);
757 memset(l, 0, sizeof(struct ha_spinlock));
758}
759
760static inline void __spin_lock(enum lock_label lbl, struct ha_spinlock *l,
761 const char *func, const char *file, int line)
762{
763 uint64_t start_time;
764
765 if (unlikely(l->info.owner & tid_bit)) {
766 /* the thread is already owning the lock */
767 abort();
768 }
769
770 HA_ATOMIC_OR(&l->info.waiters, tid_bit);
771
772 start_time = nsec_now();
773 __SPIN_LOCK(&l->lock);
774 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_write, (nsec_now() - start_time));
775
776 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
777
778
779 l->info.owner = tid_bit;
780 l->info.last_location.function = func;
781 l->info.last_location.file = file;
782 l->info.last_location.line = line;
783
784 HA_ATOMIC_AND(&l->info.waiters, ~tid_bit);
785}
786
787static inline int __spin_trylock(enum lock_label lbl, struct ha_spinlock *l,
788 const char *func, const char *file, int line)
789{
790 int r;
791
792 if (unlikely(l->info.owner & tid_bit)) {
793 /* the thread is already owning the lock */
794 abort();
795 }
796
797 /* try read should never wait */
798 r = __SPIN_TRYLOCK(&l->lock);
799 if (unlikely(r))
800 return r;
801 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
802
803 l->info.owner = tid_bit;
804 l->info.last_location.function = func;
805 l->info.last_location.file = file;
806 l->info.last_location.line = line;
807
808 return 0;
809}
810
811static inline void __spin_unlock(enum lock_label lbl, struct ha_spinlock *l,
812 const char *func, const char *file, int line)
813{
814 if (unlikely(!(l->info.owner & tid_bit))) {
815 /* the thread is not owning the lock */
816 abort();
817 }
818
819 l->info.owner = 0;
820 l->info.last_location.function = func;
821 l->info.last_location.file = file;
822 l->info.last_location.line = line;
823
Willy Tarreau7c2a2ad2017-11-02 16:26:02 +0100824 __SPIN_UNLOCK(&l->lock);
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200825 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_unlocked, 1);
826}
827
828#else /* DEBUG_THREAD */
829
830#define HA_SPINLOCK_T unsigned long
831
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100832#define HA_SPIN_INIT(l) ({ (*l) = 0; })
833#define HA_SPIN_DESTROY(l) ({ (*l) = 0; })
834#define HA_SPIN_LOCK(lbl, l) pl_take_s(l)
835#define HA_SPIN_TRYLOCK(lbl, l) !pl_try_s(l)
836#define HA_SPIN_UNLOCK(lbl, l) pl_drop_s(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200837
838#define HA_RWLOCK_T unsigned long
839
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100840#define HA_RWLOCK_INIT(l) ({ (*l) = 0; })
841#define HA_RWLOCK_DESTROY(l) ({ (*l) = 0; })
842#define HA_RWLOCK_WRLOCK(lbl,l) pl_take_w(l)
843#define HA_RWLOCK_TRYWRLOCK(lbl,l) !pl_try_w(l)
844#define HA_RWLOCK_WRUNLOCK(lbl,l) pl_drop_w(l)
845#define HA_RWLOCK_RDLOCK(lbl,l) pl_take_r(l)
846#define HA_RWLOCK_TRYRDLOCK(lbl,l) !pl_try_r(l)
847#define HA_RWLOCK_RDUNLOCK(lbl,l) pl_drop_r(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200848
849#endif /* DEBUG_THREAD */
850
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100851#ifdef __x86_64__
Willy Tarreau2325d8a2018-10-10 18:29:23 +0200852
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100853static __inline int
854__ha_cas_dw(void *target, void *compare, const void *set)
855{
856 char ret;
857
858 __asm __volatile("lock cmpxchg16b %0; setz %3"
859 : "+m" (*(void **)target),
860 "=a" (((void **)compare)[0]),
861 "=d" (((void **)compare)[1]),
862 "=q" (ret)
863 : "a" (((void **)compare)[0]),
864 "d" (((void **)compare)[1]),
865 "b" (((const void **)set)[0]),
866 "c" (((const void **)set)[1])
867 : "memory", "cc");
868 return (ret);
869}
870
871static __inline void
872__ha_barrier_load(void)
873{
874 __asm __volatile("lfence" ::: "memory");
875}
876
877static __inline void
878__ha_barrier_store(void)
879{
880 __asm __volatile("sfence" ::: "memory");
881}
882
883static __inline void
884__ha_barrier_full(void)
885{
886 __asm __volatile("mfence" ::: "memory");
887}
888
889#elif defined(__arm__) && (defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__))
Willy Tarreau2325d8a2018-10-10 18:29:23 +0200890
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100891static __inline void
892__ha_barrier_load(void)
893{
894 __asm __volatile("dmb" ::: "memory");
895}
896
897static __inline void
898__ha_barrier_store(void)
899{
900 __asm __volatile("dsb" ::: "memory");
901}
902
903static __inline void
904__ha_barrier_full(void)
905{
906 __asm __volatile("dmb" ::: "memory");
907}
908
Willy Tarreau41ccb192018-02-14 14:16:28 +0100909static __inline int __ha_cas_dw(void *target, void *compare, const void *set)
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100910{
911 uint64_t previous;
912 int tmp;
913
914 __asm __volatile("1:"
915 "ldrexd %0, [%4];"
916 "cmp %Q0, %Q2;"
917 "ittt eq;"
918 "cmpeq %R0, %R2;"
919 "strexdeq %1, %3, [%4];"
920 "cmpeq %1, #1;"
921 "beq 1b;"
922 : "=&r" (previous), "=&r" (tmp)
Willy Tarreau41ccb192018-02-14 14:16:28 +0100923 : "r" (*(uint64_t *)compare), "r" (*(uint64_t *)set), "r" (target)
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100924 : "memory", "cc");
925 tmp = (previous == *(uint64_t *)compare);
926 *(uint64_t *)compare = previous;
927 return (tmp);
928}
929
930#elif defined (__aarch64__)
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100931
932static __inline void
933__ha_barrier_load(void)
934{
935 __asm __volatile("dmb ishld" ::: "memory");
936}
937
938static __inline void
939__ha_barrier_store(void)
940{
941 __asm __volatile("dmb ishst" ::: "memory");
942}
943
944static __inline void
945__ha_barrier_full(void)
946{
947 __asm __volatile("dmb ish" ::: "memory");
948}
949
950static __inline int __ha_cas_dw(void *target, void *compare, void *set)
951{
952 void *value[2];
953 uint64_t tmp1, tmp2;
954
955 __asm__ __volatile__("1:"
956 "ldxp %0, %1, [%4];"
957 "mov %2, %0;"
958 "mov %3, %1;"
959 "eor %0, %0, %5;"
960 "eor %1, %1, %6;"
961 "orr %1, %0, %1;"
962 "mov %w0, #0;"
963 "cbnz %1, 2f;"
964 "stxp %w0, %7, %8, [%4];"
965 "cbnz %w0, 1b;"
966 "mov %w0, #1;"
967 "2:"
968 : "=&r" (tmp1), "=&r" (tmp2), "=&r" (value[0]), "=&r" (value[1])
969 : "r" (target), "r" (((void **)(compare))[0]), "r" (((void **)(compare))[1]), "r" (((void **)(set))[0]), "r" (((void **)(set))[1])
970 : "cc", "memory");
971
972 memcpy(compare, &value, sizeof(value));
973 return (tmp1);
974}
975
976#else
977#define __ha_barrier_load __sync_synchronize
978#define __ha_barrier_store __sync_synchronize
979#define __ha_barrier_full __sync_synchronize
980#endif
981
Willy Tarreaua8ae77d2018-11-25 19:28:23 +0100982void ha_spin_init(HA_SPINLOCK_T *l);
983void ha_rwlock_init(HA_RWLOCK_T *l);
984
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200985#endif /* USE_THREAD */
986
Willy Tarreau149ab772019-01-26 14:27:06 +0100987extern int thread_cpus_enabled_at_boot;
988
Olivier Houchardf61f0cb2017-12-21 17:13:05 +0100989static inline void __ha_compiler_barrier(void)
990{
991 __asm __volatile("" ::: "memory");
992}
993
Willy Tarreau0ccd3222018-07-30 10:34:35 +0200994int parse_nbthread(const char *arg, char **err);
Willy Tarreau149ab772019-01-26 14:27:06 +0100995int thread_get_default_count();
Willy Tarreau4037a3f2018-03-28 18:06:47 +0200996
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200997#endif /* _COMMON_HATHREADS_H */