blob: 9782ca9a07158d122a60ea6fe5c077660fa374ab [file] [log] [blame]
Christopher Faulet1a2b56e2017-10-12 16:09:09 +02001/*
2 * include/common/hathreads.h
3 * definitions, macros and inline functions about threads.
4 *
5 * Copyright (C) 2017 Christopher Fauet - cfaulet@haproxy.com
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation, version 2.1
10 * exclusively.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22#ifndef _COMMON_HATHREADS_H
23#define _COMMON_HATHREADS_H
24
25#include <common/config.h>
26
27#define MAX_THREADS_MASK ((unsigned long)-1)
28extern THREAD_LOCAL unsigned int tid; /* The thread id */
Christopher Faulete9a896e2017-11-14 10:16:04 +010029extern THREAD_LOCAL unsigned long tid_bit; /* The bit corresponding to the thread id */
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020030
31#ifndef USE_THREAD
32
Christopher Faulet9dcf9b62017-11-13 10:34:01 +010033#define __decl_hathreads(decl)
34
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020035#define HA_ATOMIC_CAS(val, old, new) ({((*val) == (*old)) ? (*(val) = (new) , 1) : (*(old) = *(val), 0);})
36#define HA_ATOMIC_ADD(val, i) ({*(val) += (i);})
37#define HA_ATOMIC_SUB(val, i) ({*(val) -= (i);})
38#define HA_ATOMIC_AND(val, flags) ({*(val) &= (flags);})
39#define HA_ATOMIC_OR(val, flags) ({*(val) |= (flags);})
40#define HA_ATOMIC_XCHG(val, new) \
41 ({ \
42 typeof(*(val)) __old = *(val); \
43 *(val) = new; \
44 __old; \
45 })
46#define HA_ATOMIC_STORE(val, new) ({*(val) = new;})
47#define HA_ATOMIC_UPDATE_MAX(val, new) \
48 ({ \
49 typeof(*(val)) __new = (new); \
50 \
51 if (*(val) < __new) \
52 *(val) = __new; \
53 *(val); \
54 })
55
56#define HA_ATOMIC_UPDATE_MIN(val, new) \
57 ({ \
58 typeof(*(val)) __new = (new); \
59 \
60 if (*(val) > __new) \
61 *(val) = __new; \
62 *(val); \
63 })
64
Willy Tarreaub29dc952017-10-31 18:00:20 +010065#define HA_BARRIER() do { } while (0)
Christopher Faulet339fff82017-10-19 11:59:15 +020066
67#define THREAD_SYNC_INIT(m) do { /* do nothing */ } while(0)
68#define THREAD_SYNC_ENABLE() do { /* do nothing */ } while(0)
69#define THREAD_WANT_SYNC() do { /* do nothing */ } while(0)
70#define THREAD_ENTER_SYNC() do { /* do nothing */ } while(0)
71#define THREAD_EXIT_SYNC() do { /* do nothing */ } while(0)
72#define THREAD_NO_SYNC() ({ 0; })
73#define THREAD_NEED_SYNC() ({ 1; })
74
Christopher Faulet2a944ee2017-11-07 10:42:54 +010075#define HA_SPIN_INIT(l) do { /* do nothing */ } while(0)
76#define HA_SPIN_DESTROY(l) do { /* do nothing */ } while(0)
77#define HA_SPIN_LOCK(lbl, l) do { /* do nothing */ } while(0)
78#define HA_SPIN_TRYLOCK(lbl, l) ({ 0; })
79#define HA_SPIN_UNLOCK(lbl, l) do { /* do nothing */ } while(0)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020080
Christopher Faulet2a944ee2017-11-07 10:42:54 +010081#define HA_RWLOCK_INIT(l) do { /* do nothing */ } while(0)
82#define HA_RWLOCK_DESTROY(l) do { /* do nothing */ } while(0)
83#define HA_RWLOCK_WRLOCK(lbl, l) do { /* do nothing */ } while(0)
84#define HA_RWLOCK_TRYWRLOCK(lbl, l) ({ 0; })
85#define HA_RWLOCK_WRUNLOCK(lbl, l) do { /* do nothing */ } while(0)
86#define HA_RWLOCK_RDLOCK(lbl, l) do { /* do nothing */ } while(0)
87#define HA_RWLOCK_TRYRDLOCK(lbl, l) ({ 0; })
88#define HA_RWLOCK_RDUNLOCK(lbl, l) do { /* do nothing */ } while(0)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020089
90#else /* USE_THREAD */
91
92#include <stdio.h>
93#include <stdlib.h>
94#include <string.h>
95#include <pthread.h>
96#include <import/plock.h>
97
Christopher Faulet9dcf9b62017-11-13 10:34:01 +010098#define __decl_hathreads(decl) decl
99
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200100/* TODO: thread: For now, we rely on GCC builtins but it could be a good idea to
101 * have a header file regrouping all functions dealing with threads. */
102#define HA_ATOMIC_CAS(val, old, new) __atomic_compare_exchange_n(val, old, new, 0, 0, 0)
103#define HA_ATOMIC_ADD(val, i) __atomic_add_fetch(val, i, 0)
104#define HA_ATOMIC_SUB(val, i) __atomic_sub_fetch(val, i, 0)
105#define HA_ATOMIC_AND(val, flags) __atomic_and_fetch(val, flags, 0)
106#define HA_ATOMIC_OR(val, flags) __atomic_or_fetch(val, flags, 0)
107#define HA_ATOMIC_XCHG(val, new) __atomic_exchange_n(val, new, 0)
108#define HA_ATOMIC_STORE(val, new) __atomic_store_n(val, new, 0)
109#define HA_ATOMIC_UPDATE_MAX(val, new) \
110 ({ \
111 typeof(*(val)) __old = *(val); \
112 typeof(*(val)) __new = (new); \
113 \
114 while (__old < __new && !HA_ATOMIC_CAS(val, &__old, __new)); \
115 (*val); \
116 })
117#define HA_ATOMIC_UPDATE_MIN(val, new) \
118 ({ \
119 typeof((*val)) __old = *(val); \
120 typeof((*val)) __new = (new); \
121 \
122 while (__old > __new && !HA_ATOMIC_CAS(val, &__old, __new)); \
123 (*val); \
124 })
125
Willy Tarreaub29dc952017-10-31 18:00:20 +0100126#define HA_BARRIER() pl_barrier()
127
Christopher Faulet339fff82017-10-19 11:59:15 +0200128#define THREAD_SYNC_INIT(m) thread_sync_init(m)
129#define THREAD_SYNC_ENABLE() thread_sync_enable()
130#define THREAD_WANT_SYNC() thread_want_sync()
131#define THREAD_ENTER_SYNC() thread_enter_sync()
132#define THREAD_EXIT_SYNC() thread_exit_sync()
133#define THREAD_NO_SYNC() thread_no_sync()
134#define THREAD_NEED_SYNC() thread_need_sync()
135
136int thread_sync_init(unsigned long mask);
137void thread_sync_enable(void);
138void thread_want_sync(void);
139void thread_enter_sync(void);
140void thread_exit_sync(void);
141int thread_no_sync(void);
142int thread_need_sync(void);
143
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200144#if defined(DEBUG_THREAD) || defined(DEBUG_FULL)
145
146enum lock_label {
Christopher Faulet339fff82017-10-19 11:59:15 +0200147 THREAD_SYNC_LOCK = 0,
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200148 FDTAB_LOCK,
149 FDCACHE_LOCK,
150 FD_LOCK,
151 POLL_LOCK,
Emeric Brunc60def82017-09-27 14:59:38 +0200152 TASK_RQ_LOCK,
153 TASK_WQ_LOCK,
Christopher Fauletb349e482017-08-29 09:52:38 +0200154 POOL_LOCK,
Christopher Faulet8d8aa0d2017-05-30 15:36:50 +0200155 LISTENER_LOCK,
156 LISTENER_QUEUE_LOCK,
Christopher Fauletff8abcd2017-06-02 15:33:24 +0200157 PROXY_LOCK,
Christopher Faulet29f77e82017-06-08 14:04:45 +0200158 SERVER_LOCK,
Christopher Faulet5d42e092017-10-16 12:00:40 +0200159 UPDATED_SERVERS_LOCK,
Christopher Faulet5b517552017-06-09 14:17:53 +0200160 LBPRM_LOCK,
Christopher Fauletb79a94c2017-05-30 15:34:30 +0200161 SIGNALS_LOCK,
Emeric Brun819fc6f2017-06-13 19:37:32 +0200162 STK_TABLE_LOCK,
163 STK_SESS_LOCK,
Emeric Brun1138fd02017-06-19 12:38:55 +0200164 APPLETS_LOCK,
Emeric Brun80527f52017-06-19 17:46:37 +0200165 PEER_LOCK,
Emeric Bruna1dd2432017-06-21 15:42:52 +0200166 BUF_WQ_LOCK,
Emeric Brun6b35e9b2017-06-30 16:23:45 +0200167 STRMS_LOCK,
Emeric Brun821bb9b2017-06-15 16:37:39 +0200168 SSL_LOCK,
169 SSL_GEN_CERTS_LOCK,
Emeric Brunb5997f72017-07-03 11:34:05 +0200170 PATREF_LOCK,
171 PATEXP_LOCK,
172 PATLRU_LOCK,
Christopher Faulete95f2c32017-07-24 16:30:34 +0200173 VARS_LOCK,
Christopher Faulet8ca3b4b2017-07-25 11:07:15 +0200174 COMP_POOL_LOCK,
Thierry FOURNIER61ba0e22017-07-12 11:41:21 +0200175 LUA_LOCK,
Thierry FOURNIER738a6d72017-07-17 00:14:07 +0200176 NOTIF_LOCK,
Christopher Faulet24289f22017-09-25 14:48:02 +0200177 SPOE_APPLET_LOCK,
Christopher Fauletb2812a62017-10-04 16:17:58 +0200178 DNS_LOCK,
Christopher Fauletcfda8472017-10-20 15:40:23 +0200179 PID_LIST_LOCK,
Christopher Fauletc2a89a62017-10-23 15:54:24 +0200180 EMAIL_ALERTS_LOCK,
Emeric Brund8b3b652017-11-07 11:19:48 +0100181 PIPES_LOCK,
Christopher Faulet339fff82017-10-19 11:59:15 +0200182 LOCK_LABELS
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200183};
184struct lock_stat {
185 uint64_t nsec_wait_for_write;
186 uint64_t nsec_wait_for_read;
187 uint64_t num_write_locked;
188 uint64_t num_write_unlocked;
189 uint64_t num_read_locked;
190 uint64_t num_read_unlocked;
191};
192
193extern struct lock_stat lock_stats[LOCK_LABELS];
194
195#define __HA_SPINLOCK_T unsigned long
196
197#define __SPIN_INIT(l) ({ (*l) = 0; })
198#define __SPIN_DESTROY(l) ({ (*l) = 0; })
Willy Tarreau88ac59b2017-11-06 01:03:26 +0100199#define __SPIN_LOCK(l) pl_take_s(l)
200#define __SPIN_TRYLOCK(l) !pl_try_s(l)
201#define __SPIN_UNLOCK(l) pl_drop_s(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200202
203#define __HA_RWLOCK_T unsigned long
204
205#define __RWLOCK_INIT(l) ({ (*l) = 0; })
206#define __RWLOCK_DESTROY(l) ({ (*l) = 0; })
207#define __RWLOCK_WRLOCK(l) pl_take_w(l)
208#define __RWLOCK_TRYWRLOCK(l) !pl_try_w(l)
209#define __RWLOCK_WRUNLOCK(l) pl_drop_w(l)
210#define __RWLOCK_RDLOCK(l) pl_take_r(l)
211#define __RWLOCK_TRYRDLOCK(l) !pl_try_r(l)
212#define __RWLOCK_RDUNLOCK(l) pl_drop_r(l)
213
214#define HA_SPINLOCK_T struct ha_spinlock
215
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100216#define HA_SPIN_INIT(l) __spin_init(l)
217#define HA_SPIN_DESTROY(l) __spin_destroy(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200218
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100219#define HA_SPIN_LOCK(lbl, l) __spin_lock(lbl, l, __func__, __FILE__, __LINE__)
220#define HA_SPIN_TRYLOCK(lbl, l) __spin_trylock(lbl, l, __func__, __FILE__, __LINE__)
221#define HA_SPIN_UNLOCK(lbl, l) __spin_unlock(lbl, l, __func__, __FILE__, __LINE__)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200222
223#define HA_RWLOCK_T struct ha_rwlock
224
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100225#define HA_RWLOCK_INIT(l) __ha_rwlock_init((l))
226#define HA_RWLOCK_DESTROY(l) __ha_rwlock_destroy((l))
227#define HA_RWLOCK_WRLOCK(lbl,l) __ha_rwlock_wrlock(lbl, l, __func__, __FILE__, __LINE__)
228#define HA_RWLOCK_TRYWRLOCK(lbl,l) __ha_rwlock_trywrlock(lbl, l, __func__, __FILE__, __LINE__)
229#define HA_RWLOCK_WRUNLOCK(lbl,l) __ha_rwlock_wrunlock(lbl, l, __func__, __FILE__, __LINE__)
230#define HA_RWLOCK_RDLOCK(lbl,l) __ha_rwlock_rdlock(lbl, l)
231#define HA_RWLOCK_TRYRDLOCK(lbl,l) __ha_rwlock_tryrdlock(lbl, l)
232#define HA_RWLOCK_RDUNLOCK(lbl,l) __ha_rwlock_rdunlock(lbl, l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200233
234struct ha_spinlock {
235 __HA_SPINLOCK_T lock;
236 struct {
237 unsigned long owner; /* a bit is set to 1 << tid for the lock owner */
238 unsigned long waiters; /* a bit is set to 1 << tid for waiting threads */
239 struct {
240 const char *function;
241 const char *file;
242 int line;
243 } last_location; /* location of the last owner */
244 } info;
245};
246
247struct ha_rwlock {
248 __HA_RWLOCK_T lock;
249 struct {
250 unsigned long cur_writer; /* a bit is set to 1 << tid for the lock owner */
251 unsigned long wait_writers; /* a bit is set to 1 << tid for waiting writers */
252 unsigned long cur_readers; /* a bit is set to 1 << tid for current readers */
253 unsigned long wait_readers; /* a bit is set to 1 << tid for waiting waiters */
254 struct {
255 const char *function;
256 const char *file;
257 int line;
258 } last_location; /* location of the last write owner */
259 } info;
260};
261
262static inline void show_lock_stats()
263{
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200264 const char *labels[LOCK_LABELS] = {"THREAD_SYNC", "FDTAB", "FDCACHE", "FD", "POLL",
Christopher Fauletb79a94c2017-05-30 15:34:30 +0200265 "TASK_RQ", "TASK_WQ", "POOL",
Christopher Faulet29f77e82017-06-08 14:04:45 +0200266 "LISTENER", "LISTENER_QUEUE", "PROXY", "SERVER",
Emeric Brun1138fd02017-06-19 12:38:55 +0200267 "UPDATED_SERVERS", "LBPRM", "SIGNALS", "STK_TABLE", "STK_SESS",
Emeric Brunb5997f72017-07-03 11:34:05 +0200268 "APPLETS", "PEER", "BUF_WQ", "STREAMS", "SSL", "SSL_GEN_CERTS",
Thierry FOURNIER738a6d72017-07-17 00:14:07 +0200269 "PATREF", "PATEXP", "PATLRU", "VARS", "COMP_POOL", "LUA",
Emeric Brund8b3b652017-11-07 11:19:48 +0100270 "NOTIF", "SPOE_APPLET", "DNS", "PID_LIST", "EMAIL_ALERTS",
271 "PIPES" };
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200272 int lbl;
273
274 for (lbl = 0; lbl < LOCK_LABELS; lbl++) {
275 fprintf(stderr,
276 "Stats about Lock %s: \n"
277 "\t # write lock : %lu\n"
278 "\t # write unlock: %lu (%ld)\n"
279 "\t # wait time for write : %.3f msec\n"
280 "\t # wait time for write/lock: %.3f nsec\n"
281 "\t # read lock : %lu\n"
282 "\t # read unlock : %lu (%ld)\n"
283 "\t # wait time for read : %.3f msec\n"
284 "\t # wait time for read/lock : %.3f nsec\n",
285 labels[lbl],
286 lock_stats[lbl].num_write_locked,
287 lock_stats[lbl].num_write_unlocked,
288 lock_stats[lbl].num_write_unlocked - lock_stats[lbl].num_write_locked,
289 (double)lock_stats[lbl].nsec_wait_for_write / 1000000.0,
290 lock_stats[lbl].num_write_locked ? ((double)lock_stats[lbl].nsec_wait_for_write / (double)lock_stats[lbl].num_write_locked) : 0,
291 lock_stats[lbl].num_read_locked,
292 lock_stats[lbl].num_read_unlocked,
293 lock_stats[lbl].num_read_unlocked - lock_stats[lbl].num_read_locked,
294 (double)lock_stats[lbl].nsec_wait_for_read / 1000000.0,
295 lock_stats[lbl].num_read_locked ? ((double)lock_stats[lbl].nsec_wait_for_read / (double)lock_stats[lbl].num_read_locked) : 0);
296 }
297}
298
299/* Following functions are used to collect some stats about locks. We wrap
300 * pthread functions to known how much time we wait in a lock. */
301
302static uint64_t nsec_now(void) {
303 struct timespec ts;
304
305 clock_gettime(CLOCK_MONOTONIC, &ts);
306 return ((uint64_t) ts.tv_sec * 1000000000ULL +
307 (uint64_t) ts.tv_nsec);
308}
309
310static inline void __ha_rwlock_init(struct ha_rwlock *l)
311{
312 memset(l, 0, sizeof(struct ha_rwlock));
313 __RWLOCK_INIT(&l->lock);
314}
315
316static inline void __ha_rwlock_destroy(struct ha_rwlock *l)
317{
318 __RWLOCK_DESTROY(&l->lock);
319 memset(l, 0, sizeof(struct ha_rwlock));
320}
321
322
323static inline void __ha_rwlock_wrlock(enum lock_label lbl, struct ha_rwlock *l,
324 const char *func, const char *file, int line)
325{
326 uint64_t start_time;
327
328 if (unlikely(l->info.cur_writer & tid_bit)) {
329 /* the thread is already owning the lock for write */
330 abort();
331 }
332
333 if (unlikely(l->info.cur_readers & tid_bit)) {
334 /* the thread is already owning the lock for read */
335 abort();
336 }
337
338 HA_ATOMIC_OR(&l->info.wait_writers, tid_bit);
339
340 start_time = nsec_now();
341 __RWLOCK_WRLOCK(&l->lock);
342 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_write, (nsec_now() - start_time));
343
344 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
345
346 l->info.cur_writer = tid_bit;
347 l->info.last_location.function = func;
348 l->info.last_location.file = file;
349 l->info.last_location.line = line;
350
351 HA_ATOMIC_AND(&l->info.wait_writers, ~tid_bit);
352}
353
354static inline int __ha_rwlock_trywrlock(enum lock_label lbl, struct ha_rwlock *l,
355 const char *func, const char *file, int line)
356{
357 uint64_t start_time;
358 int r;
359
360 if (unlikely(l->info.cur_writer & tid_bit)) {
361 /* the thread is already owning the lock for write */
362 abort();
363 }
364
365 if (unlikely(l->info.cur_readers & tid_bit)) {
366 /* the thread is already owning the lock for read */
367 abort();
368 }
369
370 /* We set waiting writer because trywrlock could wait for readers to quit */
371 HA_ATOMIC_OR(&l->info.wait_writers, tid_bit);
372
373 start_time = nsec_now();
374 r = __RWLOCK_TRYWRLOCK(&l->lock);
375 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_write, (nsec_now() - start_time));
376 if (unlikely(r)) {
377 HA_ATOMIC_AND(&l->info.wait_writers, ~tid_bit);
378 return r;
379 }
380 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
381
382 l->info.cur_writer = tid_bit;
383 l->info.last_location.function = func;
384 l->info.last_location.file = file;
385 l->info.last_location.line = line;
386
387 HA_ATOMIC_AND(&l->info.wait_writers, ~tid_bit);
388
389 return 0;
390}
391
392static inline void __ha_rwlock_wrunlock(enum lock_label lbl,struct ha_rwlock *l,
393 const char *func, const char *file, int line)
394{
395 if (unlikely(!(l->info.cur_writer & tid_bit))) {
396 /* the thread is not owning the lock for write */
397 abort();
398 }
399
400 l->info.cur_writer = 0;
401 l->info.last_location.function = func;
402 l->info.last_location.file = file;
403 l->info.last_location.line = line;
404
405 __RWLOCK_WRUNLOCK(&l->lock);
406
407 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_unlocked, 1);
408}
409
410static inline void __ha_rwlock_rdlock(enum lock_label lbl,struct ha_rwlock *l)
411{
412 uint64_t start_time;
413
414 if (unlikely(l->info.cur_writer & tid_bit)) {
415 /* the thread is already owning the lock for write */
416 abort();
417 }
418
419 if (unlikely(l->info.cur_readers & tid_bit)) {
420 /* the thread is already owning the lock for read */
421 abort();
422 }
423
424 HA_ATOMIC_OR(&l->info.wait_readers, tid_bit);
425
426 start_time = nsec_now();
427 __RWLOCK_RDLOCK(&l->lock);
428 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_read, (nsec_now() - start_time));
429 HA_ATOMIC_ADD(&lock_stats[lbl].num_read_locked, 1);
430
431 HA_ATOMIC_OR(&l->info.cur_readers, tid_bit);
432
433 HA_ATOMIC_AND(&l->info.wait_readers, ~tid_bit);
434}
435
436static inline int __ha_rwlock_tryrdlock(enum lock_label lbl,struct ha_rwlock *l)
437{
438 int r;
439
440 if (unlikely(l->info.cur_writer & tid_bit)) {
441 /* the thread is already owning the lock for write */
442 abort();
443 }
444
445 if (unlikely(l->info.cur_readers & tid_bit)) {
446 /* the thread is already owning the lock for read */
447 abort();
448 }
449
450 /* try read should never wait */
451 r = __RWLOCK_TRYRDLOCK(&l->lock);
452 if (unlikely(r))
453 return r;
454 HA_ATOMIC_ADD(&lock_stats[lbl].num_read_locked, 1);
455
456 HA_ATOMIC_OR(&l->info.cur_readers, tid_bit);
457
458 return 0;
459}
460
461static inline void __ha_rwlock_rdunlock(enum lock_label lbl,struct ha_rwlock *l)
462{
463 if (unlikely(!(l->info.cur_readers & tid_bit))) {
464 /* the thread is not owning the lock for read */
465 abort();
466 }
467
468 HA_ATOMIC_AND(&l->info.cur_readers, ~tid_bit);
469
470 __RWLOCK_RDUNLOCK(&l->lock);
471
472 HA_ATOMIC_ADD(&lock_stats[lbl].num_read_unlocked, 1);
473}
474
475static inline void __spin_init(struct ha_spinlock *l)
476{
477 memset(l, 0, sizeof(struct ha_spinlock));
478 __SPIN_INIT(&l->lock);
479}
480
481static inline void __spin_destroy(struct ha_spinlock *l)
482{
483 __SPIN_DESTROY(&l->lock);
484 memset(l, 0, sizeof(struct ha_spinlock));
485}
486
487static inline void __spin_lock(enum lock_label lbl, struct ha_spinlock *l,
488 const char *func, const char *file, int line)
489{
490 uint64_t start_time;
491
492 if (unlikely(l->info.owner & tid_bit)) {
493 /* the thread is already owning the lock */
494 abort();
495 }
496
497 HA_ATOMIC_OR(&l->info.waiters, tid_bit);
498
499 start_time = nsec_now();
500 __SPIN_LOCK(&l->lock);
501 HA_ATOMIC_ADD(&lock_stats[lbl].nsec_wait_for_write, (nsec_now() - start_time));
502
503 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
504
505
506 l->info.owner = tid_bit;
507 l->info.last_location.function = func;
508 l->info.last_location.file = file;
509 l->info.last_location.line = line;
510
511 HA_ATOMIC_AND(&l->info.waiters, ~tid_bit);
512}
513
514static inline int __spin_trylock(enum lock_label lbl, struct ha_spinlock *l,
515 const char *func, const char *file, int line)
516{
517 int r;
518
519 if (unlikely(l->info.owner & tid_bit)) {
520 /* the thread is already owning the lock */
521 abort();
522 }
523
524 /* try read should never wait */
525 r = __SPIN_TRYLOCK(&l->lock);
526 if (unlikely(r))
527 return r;
528 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_locked, 1);
529
530 l->info.owner = tid_bit;
531 l->info.last_location.function = func;
532 l->info.last_location.file = file;
533 l->info.last_location.line = line;
534
535 return 0;
536}
537
538static inline void __spin_unlock(enum lock_label lbl, struct ha_spinlock *l,
539 const char *func, const char *file, int line)
540{
541 if (unlikely(!(l->info.owner & tid_bit))) {
542 /* the thread is not owning the lock */
543 abort();
544 }
545
546 l->info.owner = 0;
547 l->info.last_location.function = func;
548 l->info.last_location.file = file;
549 l->info.last_location.line = line;
550
Willy Tarreau7c2a2ad2017-11-02 16:26:02 +0100551 __SPIN_UNLOCK(&l->lock);
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200552 HA_ATOMIC_ADD(&lock_stats[lbl].num_write_unlocked, 1);
553}
554
555#else /* DEBUG_THREAD */
556
557#define HA_SPINLOCK_T unsigned long
558
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100559#define HA_SPIN_INIT(l) ({ (*l) = 0; })
560#define HA_SPIN_DESTROY(l) ({ (*l) = 0; })
561#define HA_SPIN_LOCK(lbl, l) pl_take_s(l)
562#define HA_SPIN_TRYLOCK(lbl, l) !pl_try_s(l)
563#define HA_SPIN_UNLOCK(lbl, l) pl_drop_s(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200564
565#define HA_RWLOCK_T unsigned long
566
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100567#define HA_RWLOCK_INIT(l) ({ (*l) = 0; })
568#define HA_RWLOCK_DESTROY(l) ({ (*l) = 0; })
569#define HA_RWLOCK_WRLOCK(lbl,l) pl_take_w(l)
570#define HA_RWLOCK_TRYWRLOCK(lbl,l) !pl_try_w(l)
571#define HA_RWLOCK_WRUNLOCK(lbl,l) pl_drop_w(l)
572#define HA_RWLOCK_RDLOCK(lbl,l) pl_take_r(l)
573#define HA_RWLOCK_TRYRDLOCK(lbl,l) !pl_try_r(l)
574#define HA_RWLOCK_RDUNLOCK(lbl,l) pl_drop_r(l)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200575
576#endif /* DEBUG_THREAD */
577
578#endif /* USE_THREAD */
579
580#endif /* _COMMON_HATHREADS_H */