Gitiles
Code Review Sign In
git01.mediatek.com / haproxy / 7d84439b4859915fd9dcfa9362109c402c867b96 / . / include / import / plock.h
blob: ac10950a040da7c41f27edde514ab203a3582101 [file] [log] [blame]
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]1/* plock - progressive locks
2 *
3 * Copyright (C) 2012-2017 Willy Tarreau <w@1wt.eu>
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be
14 * included in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
18 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
20 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
21 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
23 * OTHER DEALINGS IN THE SOFTWARE.
24 */
25
26#include "atomic-ops.h"
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]27#ifdef _POSIX_PRIORITY_SCHEDULING
28#include <sched.h>
29#endif
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]30
31/* 64 bit */
32#define PLOCK64_RL_1 0x0000000000000004ULL
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]33#define PLOCK64_RL_2PL 0x00000000FFFFFFF8ULL
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]34#define PLOCK64_RL_ANY 0x00000000FFFFFFFCULL
35#define PLOCK64_SL_1 0x0000000100000000ULL
36#define PLOCK64_SL_ANY 0x0000000300000000ULL
37#define PLOCK64_WL_1 0x0000000400000000ULL
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]38#define PLOCK64_WL_2PL 0xFFFFFFF800000000ULL
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]39#define PLOCK64_WL_ANY 0xFFFFFFFC00000000ULL
40
41/* 32 bit */
42#define PLOCK32_RL_1 0x00000004
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]43#define PLOCK32_RL_2PL 0x0000FFF8
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]44#define PLOCK32_RL_ANY 0x0000FFFC
45#define PLOCK32_SL_1 0x00010000
46#define PLOCK32_SL_ANY 0x00030000
47#define PLOCK32_WL_1 0x00040000
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]48#define PLOCK32_WL_2PL 0xFFF80000
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]49#define PLOCK32_WL_ANY 0xFFFC0000
50
51/* dereferences <*p> as unsigned long without causing aliasing issues */
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]52#define pl_deref_long(p) ({ volatile unsigned long *__pl_l = (unsigned long *)(p); *__pl_l; })
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]53
54/* dereferences <*p> as unsigned int without causing aliasing issues */
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]55#define pl_deref_int(p) ({ volatile unsigned int *__pl_i = (unsigned int *)(p); *__pl_i; })
56
57/* This function waits for <lock> to release all bits covered by <mask>, and
58 * enforces an exponential backoff using CPU pauses to limit the pollution to
59 * the other threads' caches. The progression follows (1.5^N)-1, limited to
60 * 16384 iterations, which is way sufficient even for very large numbers of
Willy Tarreaub13044c2022-10-11 17:02:02 +0200[diff] [blame]61 * threads. It's possible to disable exponential backoff (EBO) for debugging
62 * purposes by setting PLOCK_DISABLE_EBO, in which case the function will be
63 * replaced with a simpler macro. This may for example be useful to more
64 * easily track callers' CPU usage. The macro was not designed to be used
65 * outside of the functions defined here.
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]66 */
Willy Tarreaub13044c2022-10-11 17:02:02 +0200[diff] [blame]67#if defined(PLOCK_DISABLE_EBO)
68#define pl_wait_unlock_long(lock, mask) \
69 ({ \
70 unsigned long _r; \
71 do { \
72 pl_cpu_relax(); \
73 _r = pl_deref_long(lock); \
74 } while (_r & mask); \
75 _r; /* return value */ \
76 })
77#else
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]78__attribute__((unused,noinline,no_instrument_function))
79static unsigned long pl_wait_unlock_long(const unsigned long *lock, const unsigned long mask)
80{
81 unsigned long ret;
82 unsigned int m = 0;
83
84 do {
85 unsigned int loops = m;
86
87#ifdef _POSIX_PRIORITY_SCHEDULING
88 if (loops >= 65536) {
89 sched_yield();
90 loops -= 32768;
91 }
92#endif
93 for (; loops >= 200; loops -= 10)
94 pl_cpu_relax();
95
96 for (; loops >= 1; loops--)
97 pl_barrier();
98
99 ret = pl_deref_long(lock);
100 if (__builtin_expect(ret & mask, 0) == 0)
101 break;
102
103 /* the below produces an exponential growth with loops to lower
104 * values and still growing. This allows competing threads to
105 * wait different times once the threshold is reached.
106 */
107 m = ((m + (m >> 1)) + 2) & 0x3ffff;
108 } while (1);
109
110 return ret;
111}
Willy Tarreaub13044c2022-10-11 17:02:02 +0200[diff] [blame]112#endif /* PLOCK_DISABLE_EBO */
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]113
114/* This function waits for <lock> to release all bits covered by <mask>, and
115 * enforces an exponential backoff using CPU pauses to limit the pollution to
116 * the other threads' caches. The progression follows (2^N)-1, limited to 255
117 * iterations, which is way sufficient even for very large numbers of threads.
118 * The function slightly benefits from size optimization under gcc, but Clang
119 * cannot do it, so it's not done here, as it doesn't make a big difference.
Willy Tarreaub13044c2022-10-11 17:02:02 +0200[diff] [blame]120 * It is possible to disable exponential backoff (EBO) for debugging purposes
121 * by setting PLOCK_DISABLE_EBO, in which case the function will be replaced
122 * with a simpler macro. This may for example be useful to more easily track
123 * callers' CPU usage. The macro was not designed to be used outside of the
124 * functions defined here.
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]125 */
Willy Tarreaub13044c2022-10-11 17:02:02 +0200[diff] [blame]126#if defined(PLOCK_DISABLE_EBO)
127#define pl_wait_unlock_int(lock, mask) \
128 ({ \
129 unsigned int _r; \
130 do { \
131 pl_cpu_relax(); \
132 _r = pl_deref_int(lock); \
133 } while (_r & mask); \
134 _r; /* return value */ \
135 })
136#else
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]137__attribute__((unused,noinline,no_instrument_function))
138static unsigned int pl_wait_unlock_int(const unsigned int *lock, const unsigned int mask)
139{
140 unsigned int ret;
141 unsigned int m = 0;
142
143 do {
144 unsigned int loops = m;
145
146#ifdef _POSIX_PRIORITY_SCHEDULING
147 if (loops >= 65536) {
148 sched_yield();
149 loops -= 32768;
150 }
151#endif
152 for (; loops >= 200; loops -= 10)
153 pl_cpu_relax();
154
155 for (; loops >= 1; loops--)
156 pl_barrier();
157
158 ret = pl_deref_int(lock);
159 if (__builtin_expect(ret & mask, 0) == 0)
160 break;
161
162 /* the below produces an exponential growth with loops to lower
163 * values and still growing. This allows competing threads to
164 * wait different times once the threshold is reached.
165 */
166 m = ((m + (m >> 1)) + 2) & 0x3ffff;
167 } while (1);
168
169 return ret;
170}
Willy Tarreaub13044c2022-10-11 17:02:02 +0200[diff] [blame]171#endif /* PLOCK_DISABLE_EBO */
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]172
173/* This function waits for <lock> to change from value <prev> and returns the
174 * new value. It enforces an exponential backoff using CPU pauses to limit the
175 * pollution to the other threads' caches. The progression follows (2^N)-1,
176 * limited to 255 iterations, which is way sufficient even for very large
177 * numbers of threads. It is designed to be called after a first test which
178 * retrieves the previous value, so it starts by waiting. The function slightly
179 * benefits from size optimization under gcc, but Clang cannot do it, so it's
180 * not done here, as it doesn't make a big difference.
181 */
182__attribute__((unused,noinline,no_instrument_function))
183static unsigned long pl_wait_new_long(const unsigned long *lock, const unsigned long prev)
184{
185 unsigned char m = 0;
186 unsigned long curr;
187
188 do {
189 unsigned char loops = m + 1;
190 m = (m << 1) + 1;
191 do {
192 pl_cpu_relax();
193 } while (__builtin_expect(--loops, 0));
194 curr = pl_deref_long(lock);
195 } while (__builtin_expect(curr == prev, 0));
196 return curr;
197}
198
199/* This function waits for <lock> to change from value <prev> and returns the
200 * new value. It enforces an exponential backoff using CPU pauses to limit the
201 * pollution to the other threads' caches. The progression follows (2^N)-1,
202 * limited to 255 iterations, which is way sufficient even for very large
203 * numbers of threads. It is designed to be called after a first test which
204 * retrieves the previous value, so it starts by waiting. The function slightly
205 * benefits from size optimization under gcc, but Clang cannot do it, so it's
206 * not done here, as it doesn't make a big difference.
207 */
208__attribute__((unused,noinline,no_instrument_function))
209static unsigned int pl_wait_new_int(const unsigned int *lock, const unsigned int prev)
210{
211 unsigned char m = 0;
212 unsigned int curr;
213
214 do {
215 unsigned char loops = m + 1;
216 m = (m << 1) + 1;
217 do {
218 pl_cpu_relax();
219 } while (__builtin_expect(--loops, 0));
220 curr = pl_deref_int(lock);
221 } while (__builtin_expect(curr == prev, 0));
222 return curr;
223}
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]224
225/* request shared read access (R), return non-zero on success, otherwise 0 */
226#define pl_try_r(lock) ( \
227 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]228 register unsigned long __pl_r = pl_deref_long(lock) & PLOCK64_WL_ANY; \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]229 pl_barrier(); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]230 if (!__builtin_expect(__pl_r, 0)) { \
231 __pl_r = pl_xadd((lock), PLOCK64_RL_1) & PLOCK64_WL_ANY; \
232 if (__builtin_expect(__pl_r, 0)) \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]233 pl_sub((lock), PLOCK64_RL_1); \
234 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]235 !__pl_r; /* return value */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]236 }) : (sizeof(*(lock)) == 4) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]237 register unsigned int __pl_r = pl_deref_int(lock) & PLOCK32_WL_ANY; \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]238 pl_barrier(); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]239 if (!__builtin_expect(__pl_r, 0)) { \
240 __pl_r = pl_xadd((lock), PLOCK32_RL_1) & PLOCK32_WL_ANY; \
241 if (__builtin_expect(__pl_r, 0)) \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]242 pl_sub((lock), PLOCK32_RL_1); \
243 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]244 !__pl_r; /* return value */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]245 }) : ({ \
246 void __unsupported_argument_size_for_pl_try_r__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]247 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
248 __unsupported_argument_size_for_pl_try_r__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]249 0; \
250 }) \
251)
252
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]253/* request shared read access (R) and wait for it. In order not to disturb a W
254 * lock waiting for all readers to leave, we first check if a W lock is held
255 * before trying to claim the R lock.
256 */
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]257#define pl_take_r(lock) \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]258 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
259 register unsigned long *__lk_r = (unsigned long *)(lock); \
260 register unsigned long __set_r = PLOCK64_RL_1; \
261 register unsigned long __msk_r = PLOCK64_WL_ANY; \
262 while (1) { \
263 if (__builtin_expect(pl_deref_long(__lk_r) & __msk_r, 0)) \
264 pl_wait_unlock_long(__lk_r, __msk_r); \
265 if (!__builtin_expect(pl_xadd(__lk_r, __set_r) & __msk_r, 0)) \
266 break; \
267 pl_sub(__lk_r, __set_r); \
268 } \
269 pl_barrier(); \
270 0; \
271 }) : (sizeof(*(lock)) == 4) ? ({ \
272 register unsigned int *__lk_r = (unsigned int *)(lock); \
273 register unsigned int __set_r = PLOCK32_RL_1; \
274 register unsigned int __msk_r = PLOCK32_WL_ANY; \
275 while (1) { \
276 if (__builtin_expect(pl_deref_int(__lk_r) & __msk_r, 0)) \
277 pl_wait_unlock_int(__lk_r, __msk_r); \
278 if (!__builtin_expect(pl_xadd(__lk_r, __set_r) & __msk_r, 0)) \
279 break; \
280 pl_sub(__lk_r, __set_r); \
281 } \
282 pl_barrier(); \
283 0; \
284 }) : ({ \
285 void __unsupported_argument_size_for_pl_take_r__(char *,int); \
286 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
287 __unsupported_argument_size_for_pl_take_r__(__FILE__,__LINE__); \
288 0; \
289 })
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]290
291/* release the read access (R) lock */
292#define pl_drop_r(lock) ( \
293 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]294 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]295 pl_sub(lock, PLOCK64_RL_1); \
296 }) : (sizeof(*(lock)) == 4) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]297 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]298 pl_sub(lock, PLOCK32_RL_1); \
299 }) : ({ \
300 void __unsupported_argument_size_for_pl_drop_r__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]301 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
302 __unsupported_argument_size_for_pl_drop_r__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]303 }) \
304)
305
306/* request a seek access (S), return non-zero on success, otherwise 0 */
307#define pl_try_s(lock) ( \
308 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]309 register unsigned long __pl_r = pl_deref_long(lock); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]310 pl_barrier(); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]311 if (!__builtin_expect(__pl_r & (PLOCK64_WL_ANY | PLOCK64_SL_ANY), 0)) { \
312 __pl_r = pl_xadd((lock), PLOCK64_SL_1 | PLOCK64_RL_1) & \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]313 (PLOCK64_WL_ANY | PLOCK64_SL_ANY); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]314 if (__builtin_expect(__pl_r, 0)) \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]315 pl_sub((lock), PLOCK64_SL_1 | PLOCK64_RL_1); \
316 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]317 !__pl_r; /* return value */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]318 }) : (sizeof(*(lock)) == 4) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]319 register unsigned int __pl_r = pl_deref_int(lock); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]320 pl_barrier(); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]321 if (!__builtin_expect(__pl_r & (PLOCK32_WL_ANY | PLOCK32_SL_ANY), 0)) { \
322 __pl_r = pl_xadd((lock), PLOCK32_SL_1 | PLOCK32_RL_1) & \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]323 (PLOCK32_WL_ANY | PLOCK32_SL_ANY); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]324 if (__builtin_expect(__pl_r, 0)) \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]325 pl_sub((lock), PLOCK32_SL_1 | PLOCK32_RL_1); \
326 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]327 !__pl_r; /* return value */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]328 }) : ({ \
329 void __unsupported_argument_size_for_pl_try_s__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]330 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
331 __unsupported_argument_size_for_pl_try_s__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]332 0; \
333 }) \
334)
335
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]336/* request a seek access (S) and wait for it. The lock is immediately claimed,
337 * and only upon failure an exponential backoff is used. S locks rarely compete
338 * with W locks so S will generally not disturb W. As the S lock may be used as
339 * a spinlock, it's important to grab it as fast as possible.
340 */
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]341#define pl_take_s(lock) \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]342 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
343 register unsigned long *__lk_r = (unsigned long *)(lock); \
344 register unsigned long __set_r = PLOCK64_SL_1 | PLOCK64_RL_1; \
345 register unsigned long __msk_r = PLOCK64_WL_ANY | PLOCK64_SL_ANY; \
346 while (1) { \
347 if (!__builtin_expect(pl_xadd(__lk_r, __set_r) & __msk_r, 0)) \
348 break; \
349 pl_sub(__lk_r, __set_r); \
350 pl_wait_unlock_long(__lk_r, __msk_r); \
351 } \
352 pl_barrier(); \
353 0; \
354 }) : (sizeof(*(lock)) == 4) ? ({ \
355 register unsigned int *__lk_r = (unsigned int *)(lock); \
356 register unsigned int __set_r = PLOCK32_SL_1 | PLOCK32_RL_1; \
357 register unsigned int __msk_r = PLOCK32_WL_ANY | PLOCK32_SL_ANY; \
358 while (1) { \
359 if (!__builtin_expect(pl_xadd(__lk_r, __set_r) & __msk_r, 0)) \
360 break; \
361 pl_sub(__lk_r, __set_r); \
362 pl_wait_unlock_int(__lk_r, __msk_r); \
363 } \
364 pl_barrier(); \
365 0; \
366 }) : ({ \
367 void __unsupported_argument_size_for_pl_take_s__(char *,int); \
368 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
369 __unsupported_argument_size_for_pl_take_s__(__FILE__,__LINE__); \
370 0; \
371 })
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]372
373/* release the seek access (S) lock */
374#define pl_drop_s(lock) ( \
375 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]376 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]377 pl_sub(lock, PLOCK64_SL_1 + PLOCK64_RL_1); \
378 }) : (sizeof(*(lock)) == 4) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]379 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]380 pl_sub(lock, PLOCK32_SL_1 + PLOCK32_RL_1); \
381 }) : ({ \
382 void __unsupported_argument_size_for_pl_drop_s__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]383 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
384 __unsupported_argument_size_for_pl_drop_s__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]385 }) \
386)
387
388/* drop the S lock and go back to the R lock */
389#define pl_stor(lock) ( \
390 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]391 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]392 pl_sub(lock, PLOCK64_SL_1); \
393 }) : (sizeof(*(lock)) == 4) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]394 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]395 pl_sub(lock, PLOCK32_SL_1); \
396 }) : ({ \
397 void __unsupported_argument_size_for_pl_stor__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]398 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
399 __unsupported_argument_size_for_pl_stor__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]400 }) \
401)
402
403/* take the W lock under the S lock */
404#define pl_stow(lock) ( \
405 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]406 register unsigned long __pl_r = pl_xadd((lock), PLOCK64_WL_1); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]407 while ((__pl_r & PLOCK64_RL_ANY) != PLOCK64_RL_1) \
408 __pl_r = pl_deref_long(lock); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]409 pl_barrier(); \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]410 }) : (sizeof(*(lock)) == 4) ? ({ \
411 register unsigned int __pl_r = pl_xadd((lock), PLOCK32_WL_1); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]412 while ((__pl_r & PLOCK32_RL_ANY) != PLOCK32_RL_1) \
413 __pl_r = pl_deref_int(lock); \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]414 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]415 }) : ({ \
416 void __unsupported_argument_size_for_pl_stow__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]417 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
418 __unsupported_argument_size_for_pl_stow__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]419 }) \
420)
421
422/* drop the W lock and go back to the S lock */
423#define pl_wtos(lock) ( \
424 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]425 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]426 pl_sub(lock, PLOCK64_WL_1); \
427 }) : (sizeof(*(lock)) == 4) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]428 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]429 pl_sub(lock, PLOCK32_WL_1); \
430 }) : ({ \
431 void __unsupported_argument_size_for_pl_wtos__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]432 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
433 __unsupported_argument_size_for_pl_wtos__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]434 }) \
435)
436
437/* drop the W lock and go back to the R lock */
438#define pl_wtor(lock) ( \
439 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]440 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]441 pl_sub(lock, PLOCK64_WL_1 | PLOCK64_SL_1); \
442 }) : (sizeof(*(lock)) == 4) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]443 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]444 pl_sub(lock, PLOCK32_WL_1 | PLOCK32_SL_1); \
445 }) : ({ \
446 void __unsupported_argument_size_for_pl_wtor__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]447 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
448 __unsupported_argument_size_for_pl_wtor__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]449 }) \
450)
451
452/* request a write access (W), return non-zero on success, otherwise 0.
453 *
454 * Below there is something important : by taking both W and S, we will cause
455 * an overflow of W at 4/5 of the maximum value that can be stored into W due
456 * to the fact that S is 2 bits, so we're effectively adding 5 to the word
457 * composed by W:S. But for all words multiple of 4 bits, the maximum value is
458 * multiple of 15 thus of 5. So the largest value we can store with all bits
459 * set to one will be met by adding 5, and then adding 5 again will place value
460 * 1 in W and value 0 in S, so we never leave W with 0. Also, even upon such an
461 * overflow, there's no risk to confuse it with an atomic lock because R is not
462 * null since it will not have overflown. For 32-bit locks, this situation
463 * happens when exactly 13108 threads try to grab the lock at once, W=1, S=0
464 * and R=13108. For 64-bit locks, it happens at 858993460 concurrent writers
465 * where W=1, S=0 and R=858993460.
466 */
467#define pl_try_w(lock) ( \
468 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]469 register unsigned long __pl_r = pl_deref_long(lock); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]470 pl_barrier(); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]471 if (!__builtin_expect(__pl_r & (PLOCK64_WL_ANY | PLOCK64_SL_ANY), 0)) { \
472 __pl_r = pl_xadd((lock), PLOCK64_WL_1 | PLOCK64_SL_1 | PLOCK64_RL_1); \
473 if (__builtin_expect(__pl_r & (PLOCK64_WL_ANY | PLOCK64_SL_ANY), 0)) { \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]474 /* a writer, seeker or atomic is present, let's leave */ \
475 pl_sub((lock), PLOCK64_WL_1 | PLOCK64_SL_1 | PLOCK64_RL_1); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]476 __pl_r &= (PLOCK64_WL_ANY | PLOCK64_SL_ANY); /* return value */\
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]477 } else { \
478 /* wait for all other readers to leave */ \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]479 while (__pl_r) \
480 __pl_r = pl_deref_long(lock) - \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]481 (PLOCK64_WL_1 | PLOCK64_SL_1 | PLOCK64_RL_1); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]482 } \
483 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]484 !__pl_r; /* return value */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]485 }) : (sizeof(*(lock)) == 4) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]486 register unsigned int __pl_r = pl_deref_int(lock); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]487 pl_barrier(); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]488 if (!__builtin_expect(__pl_r & (PLOCK32_WL_ANY | PLOCK32_SL_ANY), 0)) { \
489 __pl_r = pl_xadd((lock), PLOCK32_WL_1 | PLOCK32_SL_1 | PLOCK32_RL_1); \
490 if (__builtin_expect(__pl_r & (PLOCK32_WL_ANY | PLOCK32_SL_ANY), 0)) { \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]491 /* a writer, seeker or atomic is present, let's leave */ \
492 pl_sub((lock), PLOCK32_WL_1 | PLOCK32_SL_1 | PLOCK32_RL_1); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]493 __pl_r &= (PLOCK32_WL_ANY | PLOCK32_SL_ANY); /* return value */\
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]494 } else { \
495 /* wait for all other readers to leave */ \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]496 while (__pl_r) \
497 __pl_r = pl_deref_int(lock) - \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]498 (PLOCK32_WL_1 | PLOCK32_SL_1 | PLOCK32_RL_1); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]499 } \
500 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]501 !__pl_r; /* return value */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]502 }) : ({ \
503 void __unsupported_argument_size_for_pl_try_w__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]504 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
505 __unsupported_argument_size_for_pl_try_w__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]506 0; \
507 }) \
508)
509
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]510/* request a write access (W) and wait for it. The lock is immediately claimed,
511 * and only upon failure an exponential backoff is used.
512 */
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]513#define pl_take_w(lock) \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]514 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
515 register unsigned long *__lk_r = (unsigned long *)(lock); \
516 register unsigned long __set_r = PLOCK64_WL_1 | PLOCK64_SL_1 | PLOCK64_RL_1; \
517 register unsigned long __msk_r = PLOCK64_WL_ANY | PLOCK64_SL_ANY; \
518 register unsigned long __pl_r; \
519 while (1) { \
520 __pl_r = pl_xadd(__lk_r, __set_r); \
521 if (!__builtin_expect(__pl_r & __msk_r, 0)) \
522 break; \
523 pl_sub(__lk_r, __set_r); \
524 __pl_r = pl_wait_unlock_long(__lk_r, __msk_r); \
525 } \
526 /* wait for all other readers to leave */ \
527 while (__builtin_expect(__pl_r, 0)) \
528 __pl_r = pl_deref_long(__lk_r) - __set_r; \
529 pl_barrier(); \
530 0; \
531 }) : (sizeof(*(lock)) == 4) ? ({ \
532 register unsigned int *__lk_r = (unsigned int *)(lock); \
533 register unsigned int __set_r = PLOCK32_WL_1 | PLOCK32_SL_1 | PLOCK32_RL_1; \
534 register unsigned int __msk_r = PLOCK32_WL_ANY | PLOCK32_SL_ANY; \
535 register unsigned int __pl_r; \
536 while (1) { \
537 __pl_r = pl_xadd(__lk_r, __set_r); \
538 if (!__builtin_expect(__pl_r & __msk_r, 0)) \
539 break; \
540 pl_sub(__lk_r, __set_r); \
541 __pl_r = pl_wait_unlock_int(__lk_r, __msk_r); \
542 } \
543 /* wait for all other readers to leave */ \
544 while (__builtin_expect(__pl_r, 0)) \
545 __pl_r = pl_deref_int(__lk_r) - __set_r; \
546 pl_barrier(); \
547 0; \
548 }) : ({ \
549 void __unsupported_argument_size_for_pl_take_w__(char *,int); \
550 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
551 __unsupported_argument_size_for_pl_take_w__(__FILE__,__LINE__); \
552 0; \
553 })
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]554
555/* drop the write (W) lock entirely */
556#define pl_drop_w(lock) ( \
557 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]558 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]559 pl_sub(lock, PLOCK64_WL_1 | PLOCK64_SL_1 | PLOCK64_RL_1); \
560 }) : (sizeof(*(lock)) == 4) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]561 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]562 pl_sub(lock, PLOCK32_WL_1 | PLOCK32_SL_1 | PLOCK32_RL_1); \
563 }) : ({ \
564 void __unsupported_argument_size_for_pl_drop_w__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]565 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
566 __unsupported_argument_size_for_pl_drop_w__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]567 }) \
568)
569
570/* Try to upgrade from R to S, return non-zero on success, otherwise 0.
571 * This lock will fail if S or W are already held. In case of failure to grab
572 * the lock, it MUST NOT be retried without first dropping R, or it may never
573 * complete due to S waiting for R to leave before upgrading to W.
574 */
575#define pl_try_rtos(lock) ( \
576 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]577 register unsigned long __pl_r = pl_deref_long(lock); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]578 pl_barrier(); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]579 if (!__builtin_expect(__pl_r & (PLOCK64_WL_ANY | PLOCK64_SL_ANY), 0)) { \
580 __pl_r = pl_xadd((lock), PLOCK64_SL_1) & \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]581 (PLOCK64_WL_ANY | PLOCK64_SL_ANY); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]582 if (__builtin_expect(__pl_r, 0)) \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]583 pl_sub((lock), PLOCK64_SL_1); \
584 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]585 !__pl_r; /* return value */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]586 }) : (sizeof(*(lock)) == 4) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]587 register unsigned int __pl_r = pl_deref_int(lock); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]588 pl_barrier(); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]589 if (!__builtin_expect(__pl_r & (PLOCK32_WL_ANY | PLOCK32_SL_ANY), 0)) { \
590 __pl_r = pl_xadd((lock), PLOCK32_SL_1) & \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]591 (PLOCK32_WL_ANY | PLOCK32_SL_ANY); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]592 if (__builtin_expect(__pl_r, 0)) \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]593 pl_sub((lock), PLOCK32_SL_1); \
594 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]595 !__pl_r; /* return value */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]596 }) : ({ \
597 void __unsupported_argument_size_for_pl_try_rtos__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]598 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
599 __unsupported_argument_size_for_pl_try_rtos__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]600 0; \
601 }) \
602)
603
604
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]605/* Try to upgrade from R to W, return non-zero on success, otherwise 0.
606 * This lock will fail if S or W are already held. In case of failure to grab
607 * the lock, it MUST NOT be retried without first dropping R, or it may never
608 * complete due to S waiting for R to leave before upgrading to W. It waits for
609 * the last readers to leave.
610 */
611#define pl_try_rtow(lock) ( \
612 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
613 register unsigned long *__lk_r = (unsigned long *)(lock); \
614 register unsigned long __set_r = PLOCK64_WL_1 | PLOCK64_SL_1; \
615 register unsigned long __msk_r = PLOCK64_WL_ANY | PLOCK64_SL_ANY; \
616 register unsigned long __pl_r; \
617 pl_barrier(); \
618 while (1) { \
619 __pl_r = pl_xadd(__lk_r, __set_r); \
620 if (__builtin_expect(__pl_r & __msk_r, 0)) { \
621 if (pl_xadd(__lk_r, - __set_r)) \
622 break; /* the caller needs to drop the lock now */ \
623 continue; /* lock was released, try again */ \
624 } \
625 /* ok we're the only writer, wait for readers to leave */ \
626 while (__builtin_expect(__pl_r, 0)) \
627 __pl_r = pl_deref_long(__lk_r) - (PLOCK64_WL_1|PLOCK64_SL_1|PLOCK64_RL_1); \
628 /* now return with __pl_r = 0 */ \
629 break; \
630 } \
631 !__pl_r; /* return value */ \
632 }) : (sizeof(*(lock)) == 4) ? ({ \
633 register unsigned int *__lk_r = (unsigned int *)(lock); \
634 register unsigned int __set_r = PLOCK32_WL_1 | PLOCK32_SL_1; \
635 register unsigned int __msk_r = PLOCK32_WL_ANY | PLOCK32_SL_ANY; \
636 register unsigned int __pl_r; \
637 pl_barrier(); \
638 while (1) { \
639 __pl_r = pl_xadd(__lk_r, __set_r); \
640 if (__builtin_expect(__pl_r & __msk_r, 0)) { \
641 if (pl_xadd(__lk_r, - __set_r)) \
642 break; /* the caller needs to drop the lock now */ \
643 continue; /* lock was released, try again */ \
644 } \
645 /* ok we're the only writer, wait for readers to leave */ \
646 while (__builtin_expect(__pl_r, 0)) \
647 __pl_r = pl_deref_int(__lk_r) - (PLOCK32_WL_1|PLOCK32_SL_1|PLOCK32_RL_1); \
648 /* now return with __pl_r = 0 */ \
649 break; \
650 } \
651 !__pl_r; /* return value */ \
652 }) : ({ \
653 void __unsupported_argument_size_for_pl_try_rtow__(char *,int); \
654 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
655 __unsupported_argument_size_for_pl_try_rtow__(__FILE__,__LINE__); \
656 0; \
657 }) \
658)
659
660
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]661/* request atomic write access (A), return non-zero on success, otherwise 0.
662 * It's a bit tricky as we only use the W bits for this and want to distinguish
663 * between other atomic users and regular lock users. We have to give up if an
664 * S lock appears. It's possible that such a lock stays hidden in the W bits
665 * after an overflow, but in this case R is still held, ensuring we stay in the
666 * loop until we discover the conflict. The lock only return successfully if all
667 * readers are gone (or converted to A).
668 */
669#define pl_try_a(lock) ( \
670 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]671 register unsigned long __pl_r = pl_deref_long(lock) & PLOCK64_SL_ANY; \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]672 pl_barrier(); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]673 if (!__builtin_expect(__pl_r, 0)) { \
674 __pl_r = pl_xadd((lock), PLOCK64_WL_1); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]675 while (1) { \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]676 if (__builtin_expect(__pl_r & PLOCK64_SL_ANY, 0)) { \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]677 pl_sub((lock), PLOCK64_WL_1); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]678 break; /* return !__pl_r */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]679 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]680 __pl_r &= PLOCK64_RL_ANY; \
681 if (!__builtin_expect(__pl_r, 0)) \
682 break; /* return !__pl_r */ \
683 __pl_r = pl_deref_long(lock); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]684 } \
685 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]686 !__pl_r; /* return value */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]687 }) : (sizeof(*(lock)) == 4) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]688 register unsigned int __pl_r = pl_deref_int(lock) & PLOCK32_SL_ANY; \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]689 pl_barrier(); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]690 if (!__builtin_expect(__pl_r, 0)) { \
691 __pl_r = pl_xadd((lock), PLOCK32_WL_1); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]692 while (1) { \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]693 if (__builtin_expect(__pl_r & PLOCK32_SL_ANY, 0)) { \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]694 pl_sub((lock), PLOCK32_WL_1); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]695 break; /* return !__pl_r */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]696 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]697 __pl_r &= PLOCK32_RL_ANY; \
698 if (!__builtin_expect(__pl_r, 0)) \
699 break; /* return !__pl_r */ \
700 __pl_r = pl_deref_int(lock); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]701 } \
702 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]703 !__pl_r; /* return value */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]704 }) : ({ \
705 void __unsupported_argument_size_for_pl_try_a__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]706 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
707 __unsupported_argument_size_for_pl_try_a__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]708 0; \
709 }) \
710)
711
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]712/* request atomic write access (A) and wait for it. See comments in pl_try_a() for
713 * explanations.
714 */
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]715#define pl_take_a(lock) \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]716 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
717 register unsigned long *__lk_r = (unsigned long *)(lock); \
718 register unsigned long __set_r = PLOCK64_WL_1; \
719 register unsigned long __msk_r = PLOCK64_SL_ANY; \
720 register unsigned long __pl_r; \
721 __pl_r = pl_xadd(__lk_r, __set_r); \
722 while (__builtin_expect(__pl_r & PLOCK64_RL_ANY, 0)) { \
723 if (__builtin_expect(__pl_r & __msk_r, 0)) { \
724 pl_sub(__lk_r, __set_r); \
725 pl_wait_unlock_long(__lk_r, __msk_r); \
726 __pl_r = pl_xadd(__lk_r, __set_r); \
727 continue; \
728 } \
729 /* wait for all readers to leave or upgrade */ \
730 pl_cpu_relax(); pl_cpu_relax(); pl_cpu_relax(); \
731 __pl_r = pl_deref_long(lock); \
732 } \
733 pl_barrier(); \
734 0; \
735 }) : (sizeof(*(lock)) == 4) ? ({ \
736 register unsigned int *__lk_r = (unsigned int *)(lock); \
737 register unsigned int __set_r = PLOCK32_WL_1; \
738 register unsigned int __msk_r = PLOCK32_SL_ANY; \
739 register unsigned int __pl_r; \
740 __pl_r = pl_xadd(__lk_r, __set_r); \
741 while (__builtin_expect(__pl_r & PLOCK32_RL_ANY, 0)) { \
742 if (__builtin_expect(__pl_r & __msk_r, 0)) { \
743 pl_sub(__lk_r, __set_r); \
744 pl_wait_unlock_int(__lk_r, __msk_r); \
745 __pl_r = pl_xadd(__lk_r, __set_r); \
746 continue; \
747 } \
748 /* wait for all readers to leave or upgrade */ \
749 pl_cpu_relax(); pl_cpu_relax(); pl_cpu_relax(); \
750 __pl_r = pl_deref_int(lock); \
751 } \
752 pl_barrier(); \
753 0; \
754 }) : ({ \
755 void __unsupported_argument_size_for_pl_take_a__(char *,int); \
756 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
757 __unsupported_argument_size_for_pl_take_a__(__FILE__,__LINE__); \
758 0; \
759 })
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]760
761/* release atomic write access (A) lock */
762#define pl_drop_a(lock) ( \
763 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]764 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]765 pl_sub(lock, PLOCK64_WL_1); \
766 }) : (sizeof(*(lock)) == 4) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]767 pl_barrier(); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]768 pl_sub(lock, PLOCK32_WL_1); \
769 }) : ({ \
770 void __unsupported_argument_size_for_pl_drop_a__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]771 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
772 __unsupported_argument_size_for_pl_drop_a__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]773 }) \
774)
775
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]776/* Downgrade A to R. Inc(R), dec(W) then wait for W==0 */
777#define pl_ator(lock) ( \
778 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
779 register unsigned long *__lk_r = (unsigned long *)(lock); \
780 register unsigned long __set_r = PLOCK64_RL_1 - PLOCK64_WL_1; \
781 register unsigned long __msk_r = PLOCK64_WL_ANY; \
782 register unsigned long __pl_r = pl_xadd(__lk_r, __set_r) + __set_r; \
783 while (__builtin_expect(__pl_r & __msk_r, 0)) { \
784 __pl_r = pl_wait_unlock_long(__lk_r, __msk_r); \
785 } \
786 pl_barrier(); \
787 }) : (sizeof(*(lock)) == 4) ? ({ \
788 register unsigned int *__lk_r = (unsigned int *)(lock); \
789 register unsigned int __set_r = PLOCK32_RL_1 - PLOCK32_WL_1; \
790 register unsigned int __msk_r = PLOCK32_WL_ANY; \
791 register unsigned int __pl_r = pl_xadd(__lk_r, __set_r) + __set_r; \
792 while (__builtin_expect(__pl_r & __msk_r, 0)) { \
793 __pl_r = pl_wait_unlock_int(__lk_r, __msk_r); \
794 } \
795 pl_barrier(); \
796 }) : ({ \
797 void __unsupported_argument_size_for_pl_ator__(char *,int); \
798 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
799 __unsupported_argument_size_for_pl_ator__(__FILE__,__LINE__); \
800 }) \
801)
802
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]803/* Try to upgrade from R to A, return non-zero on success, otherwise 0.
804 * This lock will fail if S is held or appears while waiting (typically due to
805 * a previous grab that was disguised as a W due to an overflow). In case of
806 * failure to grab the lock, it MUST NOT be retried without first dropping R,
807 * or it may never complete due to S waiting for R to leave before upgrading
808 * to W. The lock succeeds once there's no more R (ie all of them have either
809 * completed or were turned to A).
810 */
811#define pl_try_rtoa(lock) ( \
812 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]813 register unsigned long __pl_r = pl_deref_long(lock) & PLOCK64_SL_ANY; \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]814 pl_barrier(); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]815 if (!__builtin_expect(__pl_r, 0)) { \
816 __pl_r = pl_xadd((lock), PLOCK64_WL_1 - PLOCK64_RL_1); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]817 while (1) { \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]818 if (__builtin_expect(__pl_r & PLOCK64_SL_ANY, 0)) { \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]819 pl_sub((lock), PLOCK64_WL_1 - PLOCK64_RL_1); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]820 break; /* return !__pl_r */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]821 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]822 __pl_r &= PLOCK64_RL_ANY; \
823 if (!__builtin_expect(__pl_r, 0)) \
824 break; /* return !__pl_r */ \
825 __pl_r = pl_deref_long(lock); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]826 } \
827 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]828 !__pl_r; /* return value */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]829 }) : (sizeof(*(lock)) == 4) ? ({ \
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]830 register unsigned int __pl_r = pl_deref_int(lock) & PLOCK32_SL_ANY; \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]831 pl_barrier(); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]832 if (!__builtin_expect(__pl_r, 0)) { \
833 __pl_r = pl_xadd((lock), PLOCK32_WL_1 - PLOCK32_RL_1); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]834 while (1) { \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]835 if (__builtin_expect(__pl_r & PLOCK32_SL_ANY, 0)) { \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]836 pl_sub((lock), PLOCK32_WL_1 - PLOCK32_RL_1); \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]837 break; /* return !__pl_r */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]838 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]839 __pl_r &= PLOCK32_RL_ANY; \
840 if (!__builtin_expect(__pl_r, 0)) \
841 break; /* return !__pl_r */ \
842 __pl_r = pl_deref_int(lock); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]843 } \
844 } \
Willy Tarreauf7ba77e2017-07-18 14:21:40 +0200[diff] [blame]845 !__pl_r; /* return value */ \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]846 }) : ({ \
847 void __unsupported_argument_size_for_pl_try_rtoa__(char *,int); \
Willy Tarreau2532bd22017-11-20 19:25:18 +0100[diff] [blame]848 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
849 __unsupported_argument_size_for_pl_try_rtoa__(__FILE__,__LINE__); \
Emeric Brun7122ab32017-07-07 10:26:46 +0200[diff] [blame]850 0; \
851 }) \
852)
Willy Tarreau688709d2022-07-29 17:53:31 +0200[diff] [blame]853
854
855/*
856 * The following operations cover the multiple writers model : U->R->J->C->A
857 */
858
859
860/* Upgrade R to J. Inc(W) then wait for R==W or S != 0 */
861#define pl_rtoj(lock) ( \
862 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
863 register unsigned long *__lk_r = (unsigned long *)(lock); \
864 register unsigned long __pl_r = pl_xadd(__lk_r, PLOCK64_WL_1) + PLOCK64_WL_1; \
865 register unsigned char __m = 0; \
866 while (!(__pl_r & PLOCK64_SL_ANY) && \
867 (__pl_r / PLOCK64_WL_1 != (__pl_r & PLOCK64_RL_ANY) / PLOCK64_RL_1)) { \
868 unsigned char __loops = __m + 1; \
869 __m = (__m << 1) + 1; \
870 do { \
871 pl_cpu_relax(); \
872 pl_cpu_relax(); \
873 } while (--__loops); \
874 __pl_r = pl_deref_long(__lk_r); \
875 } \
876 pl_barrier(); \
877 }) : (sizeof(*(lock)) == 4) ? ({ \
878 register unsigned int *__lk_r = (unsigned int *)(lock); \
879 register unsigned int __pl_r = pl_xadd(__lk_r, PLOCK32_WL_1) + PLOCK32_WL_1; \
880 register unsigned char __m = 0; \
881 while (!(__pl_r & PLOCK32_SL_ANY) && \
882 (__pl_r / PLOCK32_WL_1 != (__pl_r & PLOCK32_RL_ANY) / PLOCK32_RL_1)) { \
883 unsigned char __loops = __m + 1; \
884 __m = (__m << 1) + 1; \
885 do { \
886 pl_cpu_relax(); \
887 pl_cpu_relax(); \
888 } while (--__loops); \
889 __pl_r = pl_deref_int(__lk_r); \
890 } \
891 pl_barrier(); \
892 }) : ({ \
893 void __unsupported_argument_size_for_pl_rtoj__(char *,int); \
894 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
895 __unsupported_argument_size_for_pl_rtoj__(__FILE__,__LINE__); \
896 }) \
897)
898
899/* Upgrade J to C. Set S. Only one thread needs to do it though it's idempotent */
900#define pl_jtoc(lock) ( \
901 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
902 register unsigned long *__lk_r = (unsigned long *)(lock); \
903 register unsigned long __pl_r = pl_deref_long(__lk_r); \
904 if (!(__pl_r & PLOCK64_SL_ANY)) \
905 pl_or(__lk_r, PLOCK64_SL_1); \
906 pl_barrier(); \
907 }) : (sizeof(*(lock)) == 4) ? ({ \
908 register unsigned int *__lk_r = (unsigned int *)(lock); \
909 register unsigned int __pl_r = pl_deref_int(__lk_r); \
910 if (!(__pl_r & PLOCK32_SL_ANY)) \
911 pl_or(__lk_r, PLOCK32_SL_1); \
912 pl_barrier(); \
913 }) : ({ \
914 void __unsupported_argument_size_for_pl_jtoc__(char *,int); \
915 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
916 __unsupported_argument_size_for_pl_jtoc__(__FILE__,__LINE__); \
917 }) \
918)
919
920/* Upgrade R to C. Inc(W) then wait for R==W or S != 0 */
921#define pl_rtoc(lock) ( \
922 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
923 register unsigned long *__lk_r = (unsigned long *)(lock); \
924 register unsigned long __pl_r = pl_xadd(__lk_r, PLOCK64_WL_1) + PLOCK64_WL_1; \
925 register unsigned char __m = 0; \
926 while (__builtin_expect(!(__pl_r & PLOCK64_SL_ANY), 0)) { \
927 unsigned char __loops; \
928 if (__pl_r / PLOCK64_WL_1 == (__pl_r & PLOCK64_RL_ANY) / PLOCK64_RL_1) { \
929 pl_or(__lk_r, PLOCK64_SL_1); \
930 break; \
931 } \
932 __loops = __m + 1; \
933 __m = (__m << 1) + 1; \
934 do { \
935 pl_cpu_relax(); \
936 pl_cpu_relax(); \
937 } while (--__loops); \
938 __pl_r = pl_deref_long(__lk_r); \
939 } \
940 pl_barrier(); \
941 }) : (sizeof(*(lock)) == 4) ? ({ \
942 register unsigned int *__lk_r = (unsigned int *)(lock); \
943 register unsigned int __pl_r = pl_xadd(__lk_r, PLOCK32_WL_1) + PLOCK32_WL_1; \
944 register unsigned char __m = 0; \
945 while (__builtin_expect(!(__pl_r & PLOCK32_SL_ANY), 0)) { \
946 unsigned char __loops; \
947 if (__pl_r / PLOCK32_WL_1 == (__pl_r & PLOCK32_RL_ANY) / PLOCK32_RL_1) { \
948 pl_or(__lk_r, PLOCK32_SL_1); \
949 break; \
950 } \
951 __loops = __m + 1; \
952 __m = (__m << 1) + 1; \
953 do { \
954 pl_cpu_relax(); \
955 pl_cpu_relax(); \
956 } while (--__loops); \
957 __pl_r = pl_deref_int(__lk_r); \
958 } \
959 pl_barrier(); \
960 }) : ({ \
961 void __unsupported_argument_size_for_pl_rtoj__(char *,int); \
962 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
963 __unsupported_argument_size_for_pl_rtoj__(__FILE__,__LINE__); \
964 }) \
965)
966
967/* Drop the claim (C) lock : R--,W-- then clear S if !R */
968#define pl_drop_c(lock) ( \
969 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
970 register unsigned long *__lk_r = (unsigned long *)(lock); \
971 register unsigned long __set_r = - PLOCK64_RL_1 - PLOCK64_WL_1; \
972 register unsigned long __pl_r = pl_xadd(__lk_r, __set_r) + __set_r; \
973 if (!(__pl_r & PLOCK64_RL_ANY)) \
974 pl_and(__lk_r, ~PLOCK64_SL_1); \
975 pl_barrier(); \
976 }) : (sizeof(*(lock)) == 4) ? ({ \
977 register unsigned int *__lk_r = (unsigned int *)(lock); \
978 register unsigned int __set_r = - PLOCK32_RL_1 - PLOCK32_WL_1; \
979 register unsigned int __pl_r = pl_xadd(__lk_r, __set_r) + __set_r; \
980 if (!(__pl_r & PLOCK32_RL_ANY)) \
981 pl_and(__lk_r, ~PLOCK32_SL_1); \
982 pl_barrier(); \
983 }) : ({ \
984 void __unsupported_argument_size_for_pl_drop_c__(char *,int); \
985 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
986 __unsupported_argument_size_for_pl_drop_c__(__FILE__,__LINE__); \
987 }) \
988)
989
990/* Upgrade C to A. R-- then wait for !S or clear S if !R */
991#define pl_ctoa(lock) ( \
992 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
993 register unsigned long *__lk_r = (unsigned long *)(lock); \
994 register unsigned long __pl_r = pl_xadd(__lk_r, -PLOCK64_RL_1) - PLOCK64_RL_1; \
995 while (__pl_r & PLOCK64_SL_ANY) { \
996 if (!(__pl_r & PLOCK64_RL_ANY)) { \
997 pl_and(__lk_r, ~PLOCK64_SL_1); \
998 break; \
999 } \
1000 pl_cpu_relax(); \
1001 pl_cpu_relax(); \
1002 __pl_r = pl_deref_long(__lk_r); \
1003 } \
1004 pl_barrier(); \
1005 }) : (sizeof(*(lock)) == 4) ? ({ \
1006 register unsigned int *__lk_r = (unsigned int *)(lock); \
1007 register unsigned int __pl_r = pl_xadd(__lk_r, -PLOCK32_RL_1) - PLOCK32_RL_1; \
1008 while (__pl_r & PLOCK32_SL_ANY) { \
1009 if (!(__pl_r & PLOCK32_RL_ANY)) { \
1010 pl_and(__lk_r, ~PLOCK32_SL_1); \
1011 break; \
1012 } \
1013 pl_cpu_relax(); \
1014 pl_cpu_relax(); \
1015 __pl_r = pl_deref_int(__lk_r); \
1016 } \
1017 pl_barrier(); \
1018 }) : ({ \
1019 void __unsupported_argument_size_for_pl_ctoa__(char *,int); \
1020 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
1021 __unsupported_argument_size_for_pl_ctoa__(__FILE__,__LINE__); \
1022 }) \
1023)
1024
1025/* downgrade the atomic write access lock (A) to join (J) */
1026#define pl_atoj(lock) ( \
1027 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
1028 pl_barrier(); \
1029 pl_add(lock, PLOCK64_RL_1); \
1030 }) : (sizeof(*(lock)) == 4) ? ({ \
1031 pl_barrier(); \
1032 pl_add(lock, PLOCK32_RL_1); \
1033 }) : ({ \
1034 void __unsupported_argument_size_for_pl_atoj__(char *,int); \
1035 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
1036 __unsupported_argument_size_for_pl_atoj__(__FILE__,__LINE__); \
1037 }) \
1038)
1039
1040/* Returns non-zero if the thread calling it is the last writer, otherwise zero. It is
1041 * designed to be called before pl_drop_j(), pl_drop_c() or pl_drop_a() for operations
1042 * which need to be called only once.
1043 */
1044#define pl_last_writer(lock) ( \
1045 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
1046 !(pl_deref_long(lock) & PLOCK64_WL_2PL); \
1047 }) : (sizeof(*(lock)) == 4) ? ({ \
1048 !(pl_deref_int(lock) & PLOCK32_WL_2PL); \
1049 }) : ({ \
1050 void __unsupported_argument_size_for_pl_last_j__(char *,int); \
1051 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
1052 __unsupported_argument_size_for_pl_last_j__(__FILE__,__LINE__); \
1053 0; \
1054 }) \
1055)
1056
1057/* attempt to get an exclusive write access via the J lock and wait for it.
1058 * Only one thread may succeed in this operation. It will not conflict with
1059 * other users and will first wait for all writers to leave, then for all
1060 * readers to leave before starting. This offers a solution to obtain an
1061 * exclusive access to a shared resource in the R/J/C/A model. A concurrent
1062 * take_a() will wait for this one to finish first. Using a CAS instead of XADD
1063 * should make the operation converge slightly faster. Returns non-zero on
1064 * success otherwise 0.
1065 */
1066#define pl_try_j(lock) ( \
1067 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
1068 register unsigned long *__lk_r = (unsigned long *)(lock); \
1069 register unsigned long __set_r = PLOCK64_WL_1 | PLOCK64_RL_1; \
1070 register unsigned long __msk_r = PLOCK64_WL_ANY; \
1071 register unsigned long __pl_r; \
1072 register unsigned char __m; \
1073 pl_wait_unlock_long(__lk_r, __msk_r); \
1074 __pl_r = pl_xadd(__lk_r, __set_r) + __set_r; \
1075 /* wait for all other readers to leave */ \
1076 __m = 0; \
1077 while (__builtin_expect(__pl_r & PLOCK64_RL_2PL, 0)) { \
1078 unsigned char __loops; \
1079 /* give up on other writers */ \
1080 if (__builtin_expect(__pl_r & PLOCK64_WL_2PL, 0)) { \
1081 pl_sub(__lk_r, __set_r); \
1082 __pl_r = 0; /* failed to get the lock */ \
1083 break; \
1084 } \
1085 __loops = __m + 1; \
1086 __m = (__m << 1) + 1; \
1087 do { \
1088 pl_cpu_relax(); \
1089 pl_cpu_relax(); \
1090 } while (--__loops); \
1091 __pl_r = pl_deref_long(__lk_r); \
1092 } \
1093 pl_barrier(); \
1094 __pl_r; /* return value, cannot be null on success */ \
1095 }) : (sizeof(*(lock)) == 4) ? ({ \
1096 register unsigned int *__lk_r = (unsigned int *)(lock); \
1097 register unsigned int __set_r = PLOCK32_WL_1 | PLOCK32_RL_1; \
1098 register unsigned int __msk_r = PLOCK32_WL_ANY; \
1099 register unsigned int __pl_r; \
1100 register unsigned char __m; \
1101 pl_wait_unlock_int(__lk_r, __msk_r); \
1102 __pl_r = pl_xadd(__lk_r, __set_r) + __set_r; \
1103 /* wait for all other readers to leave */ \
1104 __m = 0; \
1105 while (__builtin_expect(__pl_r & PLOCK32_RL_2PL, 0)) { \
1106 unsigned char __loops; \
1107 /* but rollback on other writers */ \
1108 if (__builtin_expect(__pl_r & PLOCK32_WL_2PL, 0)) { \
1109 pl_sub(__lk_r, __set_r); \
1110 __pl_r = 0; /* failed to get the lock */ \
1111 break; \
1112 } \
1113 __loops = __m + 1; \
1114 __m = (__m << 1) + 1; \
1115 do { \
1116 pl_cpu_relax(); \
1117 pl_cpu_relax(); \
1118 } while (--__loops); \
1119 __pl_r = pl_deref_int(__lk_r); \
1120 } \
1121 pl_barrier(); \
1122 __pl_r; /* return value, cannot be null on success */ \
1123 }) : ({ \
1124 void __unsupported_argument_size_for_pl_try_j__(char *,int); \
1125 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
1126 __unsupported_argument_size_for_pl_try_j__(__FILE__,__LINE__); \
1127 0; \
1128 }) \
1129)
1130
1131/* request an exclusive write access via the J lock and wait for it. Only one
1132 * thread may succeed in this operation. It will not conflict with other users
1133 * and will first wait for all writers to leave, then for all readers to leave
1134 * before starting. This offers a solution to obtain an exclusive access to a
1135 * shared resource in the R/J/C/A model. A concurrent take_a() will wait for
1136 * this one to finish first. Using a CAS instead of XADD should make the
1137 * operation converge slightly faster.
1138 */
1139#define pl_take_j(lock) ( \
1140 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
1141 __label__ __retry; \
1142 register unsigned long *__lk_r = (unsigned long *)(lock); \
1143 register unsigned long __set_r = PLOCK64_WL_1 | PLOCK64_RL_1; \
1144 register unsigned long __msk_r = PLOCK64_WL_ANY; \
1145 register unsigned long __pl_r; \
1146 register unsigned char __m; \
1147 __retry: \
1148 pl_wait_unlock_long(__lk_r, __msk_r); \
1149 __pl_r = pl_xadd(__lk_r, __set_r) + __set_r; \
1150 /* wait for all other readers to leave */ \
1151 __m = 0; \
1152 while (__builtin_expect(__pl_r & PLOCK64_RL_2PL, 0)) { \
1153 unsigned char __loops; \
1154 /* but rollback on other writers */ \
1155 if (__builtin_expect(__pl_r & PLOCK64_WL_2PL, 0)) { \
1156 pl_sub(__lk_r, __set_r); \
1157 goto __retry; \
1158 } \
1159 __loops = __m + 1; \
1160 __m = (__m << 1) + 1; \
1161 do { \
1162 pl_cpu_relax(); \
1163 pl_cpu_relax(); \
1164 } while (--__loops); \
1165 __pl_r = pl_deref_long(__lk_r); \
1166 } \
1167 pl_barrier(); \
1168 0; \
1169 }) : (sizeof(*(lock)) == 4) ? ({ \
1170 __label__ __retry; \
1171 register unsigned int *__lk_r = (unsigned int *)(lock); \
1172 register unsigned int __set_r = PLOCK32_WL_1 | PLOCK32_RL_1; \
1173 register unsigned int __msk_r = PLOCK32_WL_ANY; \
1174 register unsigned int __pl_r; \
1175 register unsigned char __m; \
1176 __retry: \
1177 pl_wait_unlock_int(__lk_r, __msk_r); \
1178 __pl_r = pl_xadd(__lk_r, __set_r) + __set_r; \
1179 /* wait for all other readers to leave */ \
1180 __m = 0; \
1181 while (__builtin_expect(__pl_r & PLOCK32_RL_2PL, 0)) { \
1182 unsigned char __loops; \
1183 /* but rollback on other writers */ \
1184 if (__builtin_expect(__pl_r & PLOCK32_WL_2PL, 0)) { \
1185 pl_sub(__lk_r, __set_r); \
1186 goto __retry; \
1187 } \
1188 __loops = __m + 1; \
1189 __m = (__m << 1) + 1; \
1190 do { \
1191 pl_cpu_relax(); \
1192 pl_cpu_relax(); \
1193 } while (--__loops); \
1194 __pl_r = pl_deref_int(__lk_r); \
1195 } \
1196 pl_barrier(); \
1197 0; \
1198 }) : ({ \
1199 void __unsupported_argument_size_for_pl_take_j__(char *,int); \
1200 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
1201 __unsupported_argument_size_for_pl_take_j__(__FILE__,__LINE__); \
1202 0; \
1203 }) \
1204)
1205
1206/* drop the join (J) lock entirely */
1207#define pl_drop_j(lock) ( \
1208 (sizeof(long) == 8 && sizeof(*(lock)) == 8) ? ({ \
1209 pl_barrier(); \
1210 pl_sub(lock, PLOCK64_WL_1 | PLOCK64_RL_1); \
1211 }) : (sizeof(*(lock)) == 4) ? ({ \
1212 pl_barrier(); \
1213 pl_sub(lock, PLOCK32_WL_1 | PLOCK32_RL_1); \
1214 }) : ({ \
1215 void __unsupported_argument_size_for_pl_drop_j__(char *,int); \
1216 if (sizeof(*(lock)) != 4 && (sizeof(long) != 8 || sizeof(*(lock)) != 8)) \
1217 __unsupported_argument_size_for_pl_drop_j__(__FILE__,__LINE__); \
1218 }) \
1219)
1220
1221/*
1222 * The part below is for Low Overhead R/W locks (LORW). These ones are not
1223 * upgradable and not necessarily fair but they try to be fast when uncontended
1224 * and to limit the cost and perturbation during contention. Writers always
1225 * have precedence over readers to preserve latency as much as possible.
1226 *
1227 * The principle is to offer a fast no-contention path and a limited total
1228 * number of writes for the contended path. Since R/W locks are expected to be
1229 * used in situations where there is a benefit in separating reads from writes,
1230 * it is expected that reads are common (typ >= 50%) and that there is often at
1231 * least one reader (otherwise a spinlock wouldn't be a problem). As such, a
1232 * reader will try to pass instantly, detect contention and immediately retract
1233 * and wait in the queue in case there is contention. A writer will first also
1234 * try to pass instantly, and if it fails due to pending readers, it will mark
1235 * that it's waiting so that readers stop entering. This will leave the writer
1236 * waiting as close as possible to the point of being granted access. New
1237 * writers will also notice this previous contention and will wait outside.
1238 * This means that a successful access for a reader or a writer requires a
1239 * single CAS, and a contended attempt will require one failed CAS and one
1240 * successful XADD for a reader, or an optional OR and a N+1 CAS for the
1241 * writer.
1242 *
1243 * A counter of shared users indicates the number of active readers, while a
1244 * (single-bit) counter of exclusive writers indicates whether the lock is
1245 * currently held for writes. This distinction also permits to use a single
1246 * function to release the lock if desired, since the exclusive bit indicates
1247 * the state of the caller of unlock(). The WRQ bit is cleared during the
1248 * unlock.
1249 *
1250 * Layout: (32/64 bit):
1251 * 31 2 1 0
1252 * +-----------+--------------+-----+-----+
1253 * | | SHR | WRQ | EXC |
1254 * +-----------+--------------+-----+-----+
1255 *
1256 * In order to minimize operations, the WRQ bit is held during EXC so that the
1257 * write waiter that had to fight for EXC doesn't have to release WRQ during
1258 * its operations, and will just drop it along with EXC upon unlock.
1259 *
1260 * This means the following costs:
1261 * reader:
1262 * success: 1 CAS
1263 * failure: 1 CAS + 1 XADD
1264 * unlock: 1 SUB
1265 * writer:
1266 * success: 1 RD + 1 CAS
1267 * failure: 1 RD + 1 CAS + 0/1 OR + N CAS
1268 * unlock: 1 AND
1269 */
1270
1271#define PLOCK_LORW_EXC_BIT ((sizeof(long) == 8) ? 0 : 0)
1272#define PLOCK_LORW_EXC_SIZE ((sizeof(long) == 8) ? 1 : 1)
1273#define PLOCK_LORW_EXC_BASE (1UL << PLOCK_LORW_EXC_BIT)
1274#define PLOCK_LORW_EXC_MASK (((1UL << PLOCK_LORW_EXC_SIZE) - 1UL) << PLOCK_LORW_EXC_BIT)
1275
1276#define PLOCK_LORW_WRQ_BIT ((sizeof(long) == 8) ? 1 : 1)
1277#define PLOCK_LORW_WRQ_SIZE ((sizeof(long) == 8) ? 1 : 1)
1278#define PLOCK_LORW_WRQ_BASE (1UL << PLOCK_LORW_WRQ_BIT)
1279#define PLOCK_LORW_WRQ_MASK (((1UL << PLOCK_LORW_WRQ_SIZE) - 1UL) << PLOCK_LORW_WRQ_BIT)
1280
1281#define PLOCK_LORW_SHR_BIT ((sizeof(long) == 8) ? 2 : 2)
1282#define PLOCK_LORW_SHR_SIZE ((sizeof(long) == 8) ? 30 : 30)
1283#define PLOCK_LORW_SHR_BASE (1UL << PLOCK_LORW_SHR_BIT)
1284#define PLOCK_LORW_SHR_MASK (((1UL << PLOCK_LORW_SHR_SIZE) - 1UL) << PLOCK_LORW_SHR_BIT)
1285
1286__attribute__((unused,always_inline,no_instrument_function))
1287static inline void pl_lorw_rdlock(unsigned long *lock)
1288{
1289 unsigned long lk = 0;
1290
1291 /* First, assume we're alone and try to get the read lock (fast path).
1292 * It often works because read locks are often used on low-contention
1293 * structs.
1294 */
1295 lk = pl_cmpxchg(lock, 0, PLOCK_LORW_SHR_BASE);
1296 if (!lk)
1297 return;
1298
1299 /* so we were not alone, make sure there's no writer waiting for the
1300 * lock to be empty of visitors.
1301 */
1302 if (lk & PLOCK_LORW_WRQ_MASK)
1303 lk = pl_wait_unlock_long(lock, PLOCK_LORW_WRQ_MASK);
1304
1305 /* count us as visitor among others */
1306 lk = pl_xadd(lock, PLOCK_LORW_SHR_BASE);
1307
1308 /* wait for end of exclusive access if any */
1309 if (lk & PLOCK_LORW_EXC_MASK)
1310 lk = pl_wait_unlock_long(lock, PLOCK_LORW_EXC_MASK);
1311}
1312
1313
1314__attribute__((unused,always_inline,no_instrument_function))
1315static inline void pl_lorw_wrlock(unsigned long *lock)
1316{
1317 unsigned long lk = 0;
1318 unsigned long old = 0;
1319
1320 /* first, make sure another writer is not already blocked waiting for
1321 * readers to leave. Note that tests have shown that it can be even
1322 * faster to avoid the first check and to unconditionally wait.
1323 */
1324 lk = pl_deref_long(lock);
1325 if (__builtin_expect(lk & PLOCK_LORW_WRQ_MASK, 1))
1326 lk = pl_wait_unlock_long(lock, PLOCK_LORW_WRQ_MASK);
1327
1328 do {
1329 /* let's check for the two sources of contention at once */
1330
1331 if (__builtin_expect(lk & (PLOCK_LORW_SHR_MASK | PLOCK_LORW_EXC_MASK), 1)) {
1332 /* check if there are still readers coming. If so, close the door and
1333 * wait for them to leave.
1334 */
1335 if (lk & PLOCK_LORW_SHR_MASK) {
1336 /* note below, an OR is significantly cheaper than BTS or XADD */
1337 if (!(lk & PLOCK_LORW_WRQ_MASK))
1338 pl_or(lock, PLOCK_LORW_WRQ_BASE);
1339 lk = pl_wait_unlock_long(lock, PLOCK_LORW_SHR_MASK);
1340 }
1341
1342 /* And also wait for a previous writer to finish. */
1343 if (lk & PLOCK_LORW_EXC_MASK)
1344 lk = pl_wait_unlock_long(lock, PLOCK_LORW_EXC_MASK);
1345 }
1346
1347 /* A fresh new reader may appear right now if there were none
1348 * above and we didn't close the door.
1349 */
1350 old = lk & ~PLOCK_LORW_SHR_MASK & ~PLOCK_LORW_EXC_MASK;
1351 lk = pl_cmpxchg(lock, old, old | PLOCK_LORW_EXC_BASE);
1352 } while (lk != old);
1353
1354 /* done, not waiting anymore, the WRQ bit if any, will be dropped by the
1355 * unlock
1356 */
1357}
1358
1359
1360__attribute__((unused,always_inline,no_instrument_function))
1361static inline void pl_lorw_rdunlock(unsigned long *lock)
1362{
1363 pl_sub(lock, PLOCK_LORW_SHR_BASE);
1364}
1365
1366__attribute__((unused,always_inline,no_instrument_function))
1367static inline void pl_lorw_wrunlock(unsigned long *lock)
1368{
1369 pl_and(lock, ~(PLOCK_LORW_WRQ_MASK | PLOCK_LORW_EXC_MASK));
1370}
1371
1372__attribute__((unused,always_inline,no_instrument_function))
1373static inline void pl_lorw_unlock(unsigned long *lock)
1374{
1375 if (pl_deref_long(lock) & PLOCK_LORW_EXC_MASK)
1376 pl_lorw_wrunlock(lock);
1377 else
1378 pl_lorw_rdunlock(lock);
1379}