blob: 5eb68e33ad91b82dcb3206a8d41021616a4e8720 [file] [log] [blame]
Christopher Faulet1a2b56e2017-10-12 16:09:09 +02001/*
2 * functions about threads.
3 *
4 * Copyright (C) 2017 Christopher Fauet - cfaulet@haproxy.com
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
Willy Tarreau149ab772019-01-26 14:27:06 +010013#define _GNU_SOURCE
Christopher Faulet339fff82017-10-19 11:59:15 +020014#include <unistd.h>
Willy Tarreau0ccd3222018-07-30 10:34:35 +020015#include <stdlib.h>
Christopher Faulet339fff82017-10-19 11:59:15 +020016#include <fcntl.h>
17
Willy Tarreau149ab772019-01-26 14:27:06 +010018#ifdef USE_CPU_AFFINITY
19#include <sched.h>
20#endif
21
Olivier Houchard46453d32019-04-11 00:06:47 +020022#ifdef __FreeBSD__
23#include <sys/cpuset.h>
24#endif
25
Willy Tarreau6be78492020-06-05 00:00:29 +020026#include <haproxy/cfgparse.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020027#include <haproxy/fd.h>
28#include <haproxy/global.h>
Willy Tarreau3f567e42020-05-28 15:29:19 +020029#include <haproxy/thread.h>
Willy Tarreau48fbcae2020-06-03 18:09:46 +020030#include <haproxy/tools.h>
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020031
David Carliera92c5ce2019-09-13 05:03:12 +010032struct thread_info ha_thread_info[MAX_THREADS] = { };
33THREAD_LOCAL struct thread_info *ti = &ha_thread_info[0];
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020034
35#ifdef USE_THREAD
36
Willy Tarreau60b639c2018-08-02 10:16:17 +020037volatile unsigned long threads_want_rdv_mask = 0;
38volatile unsigned long threads_harmless_mask = 0;
Willy Tarreau9a1f5732019-06-09 12:20:02 +020039volatile unsigned long threads_sync_mask = 0;
Willy Tarreau0ccd3222018-07-30 10:34:35 +020040volatile unsigned long all_threads_mask = 1; // nbthread 1 assumed by default
Willy Tarreau0c026f42018-08-01 19:12:20 +020041THREAD_LOCAL unsigned int tid = 0;
42THREAD_LOCAL unsigned long tid_bit = (1UL << 0);
Willy Tarreau149ab772019-01-26 14:27:06 +010043int thread_cpus_enabled_at_boot = 1;
Willy Tarreau0c026f42018-08-01 19:12:20 +020044
Christopher Faulet339fff82017-10-19 11:59:15 +020045
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020046#if defined(DEBUG_THREAD) || defined(DEBUG_FULL)
47struct lock_stat lock_stats[LOCK_LABELS];
48#endif
49
Willy Tarreau60b639c2018-08-02 10:16:17 +020050/* Marks the thread as harmless until the last thread using the rendez-vous
51 * point quits. Given that we can wait for a long time, sched_yield() is used
52 * when available to offer the CPU resources to competing threads if needed.
53 */
54void thread_harmless_till_end()
55{
Olivier Houchardb23a61f2019-03-08 18:51:17 +010056 _HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
Willy Tarreau60b639c2018-08-02 10:16:17 +020057 while (threads_want_rdv_mask & all_threads_mask) {
Willy Tarreau38171da2019-05-17 16:33:13 +020058 ha_thread_relax();
Willy Tarreau60b639c2018-08-02 10:16:17 +020059 }
60}
61
62/* Isolates the current thread : request the ability to work while all other
63 * threads are harmless. Only returns once all of them are harmless, with the
64 * current thread's bit in threads_harmless_mask cleared. Needs to be completed
65 * using thread_release().
66 */
67void thread_isolate()
68{
69 unsigned long old;
70
Olivier Houchardb23a61f2019-03-08 18:51:17 +010071 _HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
72 __ha_barrier_atomic_store();
73 _HA_ATOMIC_OR(&threads_want_rdv_mask, tid_bit);
Willy Tarreau60b639c2018-08-02 10:16:17 +020074
75 /* wait for all threads to become harmless */
76 old = threads_harmless_mask;
77 while (1) {
78 if (unlikely((old & all_threads_mask) != all_threads_mask))
79 old = threads_harmless_mask;
Olivier Houchardb23a61f2019-03-08 18:51:17 +010080 else if (_HA_ATOMIC_CAS(&threads_harmless_mask, &old, old & ~tid_bit))
Willy Tarreau60b639c2018-08-02 10:16:17 +020081 break;
82
Willy Tarreau38171da2019-05-17 16:33:13 +020083 ha_thread_relax();
Willy Tarreau60b639c2018-08-02 10:16:17 +020084 }
85 /* one thread gets released at a time here, with its harmess bit off.
86 * The loss of this bit makes the other one continue to spin while the
87 * thread is working alone.
88 */
89}
90
91/* Cancels the effect of thread_isolate() by releasing the current thread's bit
92 * in threads_want_rdv_mask and by marking this thread as harmless until the
93 * last worker finishes.
94 */
95void thread_release()
96{
Olivier Houchardb23a61f2019-03-08 18:51:17 +010097 _HA_ATOMIC_AND(&threads_want_rdv_mask, ~tid_bit);
Willy Tarreau31cba0d2019-06-09 08:44:19 +020098 while (threads_want_rdv_mask & all_threads_mask) {
99 _HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
100 while (threads_want_rdv_mask & all_threads_mask)
101 ha_thread_relax();
102 HA_ATOMIC_AND(&threads_harmless_mask, ~tid_bit);
103 }
Willy Tarreau60b639c2018-08-02 10:16:17 +0200104}
Christopher Faulet339fff82017-10-19 11:59:15 +0200105
Willy Tarreau9a1f5732019-06-09 12:20:02 +0200106/* Cancels the effect of thread_isolate() by releasing the current thread's bit
107 * in threads_want_rdv_mask and by marking this thread as harmless until the
108 * last worker finishes. The difference with thread_release() is that this one
109 * will not leave the function before others are notified to do the same, so it
110 * guarantees that the current thread will not pass through a subsequent call
111 * to thread_isolate() before others finish.
112 */
113void thread_sync_release()
114{
115 _HA_ATOMIC_OR(&threads_sync_mask, tid_bit);
116 __ha_barrier_atomic_store();
117 _HA_ATOMIC_AND(&threads_want_rdv_mask, ~tid_bit);
118
119 while (threads_want_rdv_mask & all_threads_mask) {
120 _HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
121 while (threads_want_rdv_mask & all_threads_mask)
122 ha_thread_relax();
123 HA_ATOMIC_AND(&threads_harmless_mask, ~tid_bit);
124 }
125
126 /* the current thread is not harmless anymore, thread_isolate()
127 * is forced to wait till all waiters finish.
128 */
129 _HA_ATOMIC_AND(&threads_sync_mask, ~tid_bit);
130 while (threads_sync_mask & all_threads_mask)
131 ha_thread_relax();
132}
133
Willy Tarreau2beaaf72019-05-22 08:43:34 +0200134/* send signal <sig> to thread <thr> */
135void ha_tkill(unsigned int thr, int sig)
136{
David Carliera92c5ce2019-09-13 05:03:12 +0100137 pthread_kill(ha_thread_info[thr].pthread, sig);
Willy Tarreau2beaaf72019-05-22 08:43:34 +0200138}
139
140/* send signal <sig> to all threads. The calling thread is signaled last in
141 * order to allow all threads to synchronize in the handler.
142 */
143void ha_tkillall(int sig)
144{
145 unsigned int thr;
146
147 for (thr = 0; thr < global.nbthread; thr++) {
148 if (!(all_threads_mask & (1UL << thr)))
149 continue;
150 if (thr == tid)
151 continue;
David Carliera92c5ce2019-09-13 05:03:12 +0100152 pthread_kill(ha_thread_info[thr].pthread, sig);
Willy Tarreau2beaaf72019-05-22 08:43:34 +0200153 }
154 raise(sig);
155}
156
Willy Tarreaua8ae77d2018-11-25 19:28:23 +0100157/* these calls are used as callbacks at init time */
158void ha_spin_init(HA_SPINLOCK_T *l)
159{
160 HA_SPIN_INIT(l);
161}
162
163/* these calls are used as callbacks at init time */
164void ha_rwlock_init(HA_RWLOCK_T *l)
165{
166 HA_RWLOCK_INIT(l);
167}
168
Willy Tarreau149ab772019-01-26 14:27:06 +0100169/* returns the number of CPUs the current process is enabled to run on */
170static int thread_cpus_enabled()
171{
172 int ret = 1;
173
174#ifdef USE_CPU_AFFINITY
175#if defined(__linux__) && defined(CPU_COUNT)
176 cpu_set_t mask;
177
178 if (sched_getaffinity(0, sizeof(mask), &mask) == 0)
179 ret = CPU_COUNT(&mask);
Olivier Houchard46453d32019-04-11 00:06:47 +0200180#elif defined(__FreeBSD__) && defined(USE_CPU_AFFINITY)
181 cpuset_t cpuset;
182 if (cpuset_getaffinity(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1,
183 sizeof(cpuset), &cpuset) == 0)
184 ret = CPU_COUNT(&cpuset);
Willy Tarreau149ab772019-01-26 14:27:06 +0100185#endif
186#endif
187 ret = MAX(ret, 1);
188 ret = MIN(ret, MAX_THREADS);
189 return ret;
190}
191
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200192__attribute__((constructor))
Willy Tarreau3f567e42020-05-28 15:29:19 +0200193static void __thread_init(void)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200194{
Willy Tarreauf5809cd2019-01-26 13:35:03 +0100195 char *ptr = NULL;
196
197 if (MAX_THREADS < 1 || MAX_THREADS > LONGBITS) {
198 ha_alert("MAX_THREADS value must be between 1 and %d inclusive; "
199 "HAProxy was built with value %d, please fix it and rebuild.\n",
200 LONGBITS, MAX_THREADS);
201 exit(1);
202 }
Willy Tarreau149ab772019-01-26 14:27:06 +0100203
Willy Tarreau06a18062020-09-02 09:53:47 +0200204#if defined(__GNUC__) && (__GNUC__ >= 3) && defined(__GNU_LIBRARY__) && !defined(__clang__)
Willy Tarreau77b98222020-09-02 08:04:35 +0200205 /* make sure libgcc_s is already loaded, because pthread_exit() may
206 * may need it on exit after the chroot! _Unwind_Find_FDE() is defined
207 * there since gcc 3.0, has no side effect, doesn't take any argument
208 * and seems to be present on all supported platforms.
209 */
210 {
211 extern void _Unwind_Find_FDE(void);
212 _Unwind_Find_FDE();
213 }
214#endif
215
Willy Tarreau149ab772019-01-26 14:27:06 +0100216 thread_cpus_enabled_at_boot = thread_cpus_enabled();
217
218 memprintf(&ptr, "Built with multi-threading support (MAX_THREADS=%d, default=%d).",
219 MAX_THREADS, thread_cpus_enabled_at_boot);
Willy Tarreauf5809cd2019-01-26 13:35:03 +0100220 hap_register_build_opts(ptr, 1);
221
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200222#if defined(DEBUG_THREAD) || defined(DEBUG_FULL)
223 memset(lock_stats, 0, sizeof(lock_stats));
224#endif
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200225}
226
Willy Tarreau8459f252018-12-15 16:48:14 +0100227#else
228
229REGISTER_BUILD_OPTS("Built without multi-threading support (USE_THREAD not set).");
230
Willy Tarreau0ccd3222018-07-30 10:34:35 +0200231#endif // USE_THREAD
232
233
234/* Parse the number of threads in argument <arg>, returns it and adjusts a few
235 * internal variables accordingly, or fails and returns zero with an error
236 * reason in <errmsg>. May be called multiple times while parsing.
237 */
238int parse_nbthread(const char *arg, char **err)
239{
240 long nbthread;
241 char *errptr;
242
243 nbthread = strtol(arg, &errptr, 10);
244 if (!*arg || *errptr) {
245 memprintf(err, "passed a missing or unparsable integer value in '%s'", arg);
246 return 0;
247 }
248
249#ifndef USE_THREAD
250 if (nbthread != 1) {
251 memprintf(err, "specified with a value other than 1 while HAProxy is not compiled with threads support. Please check build options for USE_THREAD");
252 return 0;
253 }
254#else
255 if (nbthread < 1 || nbthread > MAX_THREADS) {
256 memprintf(err, "value must be between 1 and %d (was %ld)", MAX_THREADS, nbthread);
257 return 0;
258 }
259
Willy Tarreaufc647362019-02-02 17:05:03 +0100260 all_threads_mask = nbits(nbthread);
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200261#endif
Willy Tarreau0ccd3222018-07-30 10:34:35 +0200262 return nbthread;
263}