blob: bc655bd501508864fee70c5b09433f380a2fc3f5 [file] [log] [blame]
Christopher Faulet1a2b56e2017-10-12 16:09:09 +02001/*
2 * functions about threads.
3 *
4 * Copyright (C) 2017 Christopher Fauet - cfaulet@haproxy.com
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
Willy Tarreau149ab772019-01-26 14:27:06 +010013#define _GNU_SOURCE
Christopher Faulet339fff82017-10-19 11:59:15 +020014#include <unistd.h>
Willy Tarreau0ccd3222018-07-30 10:34:35 +020015#include <stdlib.h>
Christopher Faulet339fff82017-10-19 11:59:15 +020016#include <fcntl.h>
17
Willy Tarreau149ab772019-01-26 14:27:06 +010018#ifdef USE_CPU_AFFINITY
19#include <sched.h>
20#endif
21
Olivier Houchard46453d32019-04-11 00:06:47 +020022#ifdef __FreeBSD__
23#include <sys/cpuset.h>
24#endif
25
Willy Tarreau6be78492020-06-05 00:00:29 +020026#include <haproxy/cfgparse.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020027#include <haproxy/fd.h>
28#include <haproxy/global.h>
Willy Tarreau11bd6f72021-05-08 20:33:02 +020029#include <haproxy/log.h>
Willy Tarreau3f567e42020-05-28 15:29:19 +020030#include <haproxy/thread.h>
Willy Tarreau48fbcae2020-06-03 18:09:46 +020031#include <haproxy/tools.h>
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020032
David Carliera92c5ce2019-09-13 05:03:12 +010033struct thread_info ha_thread_info[MAX_THREADS] = { };
34THREAD_LOCAL struct thread_info *ti = &ha_thread_info[0];
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020035
36#ifdef USE_THREAD
37
Willy Tarreau56c3b8b2021-04-10 17:28:18 +020038volatile unsigned long threads_want_rdv_mask __read_mostly = 0;
Willy Tarreau60b639c2018-08-02 10:16:17 +020039volatile unsigned long threads_harmless_mask = 0;
Willy Tarreau9a1f5732019-06-09 12:20:02 +020040volatile unsigned long threads_sync_mask = 0;
Willy Tarreau56c3b8b2021-04-10 17:28:18 +020041volatile unsigned long all_threads_mask __read_mostly = 1; // nbthread 1 assumed by default
Willy Tarreau0c026f42018-08-01 19:12:20 +020042THREAD_LOCAL unsigned int tid = 0;
43THREAD_LOCAL unsigned long tid_bit = (1UL << 0);
Willy Tarreau149ab772019-01-26 14:27:06 +010044int thread_cpus_enabled_at_boot = 1;
Willy Tarreau0c026f42018-08-01 19:12:20 +020045
Christopher Faulet339fff82017-10-19 11:59:15 +020046
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020047#if defined(DEBUG_THREAD) || defined(DEBUG_FULL)
48struct lock_stat lock_stats[LOCK_LABELS];
49#endif
50
Willy Tarreau60b639c2018-08-02 10:16:17 +020051/* Marks the thread as harmless until the last thread using the rendez-vous
Christopher Fauleta9a9e9a2021-03-25 14:11:36 +010052 * point quits, excluding the current one. Thus an isolated thread may be safely
53 * marked as harmless. Given that we can wait for a long time, sched_yield() is
54 * used when available to offer the CPU resources to competing threads if
55 * needed.
Willy Tarreau60b639c2018-08-02 10:16:17 +020056 */
57void thread_harmless_till_end()
58{
Olivier Houchardb23a61f2019-03-08 18:51:17 +010059 _HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
Christopher Faulet76b44192021-04-16 11:33:39 +020060 while (threads_want_rdv_mask & all_threads_mask & ~tid_bit) {
Willy Tarreau38171da2019-05-17 16:33:13 +020061 ha_thread_relax();
Willy Tarreau60b639c2018-08-02 10:16:17 +020062 }
63}
64
65/* Isolates the current thread : request the ability to work while all other
66 * threads are harmless. Only returns once all of them are harmless, with the
67 * current thread's bit in threads_harmless_mask cleared. Needs to be completed
68 * using thread_release().
69 */
70void thread_isolate()
71{
72 unsigned long old;
73
Olivier Houchardb23a61f2019-03-08 18:51:17 +010074 _HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
75 __ha_barrier_atomic_store();
76 _HA_ATOMIC_OR(&threads_want_rdv_mask, tid_bit);
Willy Tarreau60b639c2018-08-02 10:16:17 +020077
78 /* wait for all threads to become harmless */
79 old = threads_harmless_mask;
80 while (1) {
81 if (unlikely((old & all_threads_mask) != all_threads_mask))
82 old = threads_harmless_mask;
Olivier Houchardb23a61f2019-03-08 18:51:17 +010083 else if (_HA_ATOMIC_CAS(&threads_harmless_mask, &old, old & ~tid_bit))
Willy Tarreau60b639c2018-08-02 10:16:17 +020084 break;
85
Willy Tarreau38171da2019-05-17 16:33:13 +020086 ha_thread_relax();
Willy Tarreau60b639c2018-08-02 10:16:17 +020087 }
88 /* one thread gets released at a time here, with its harmess bit off.
89 * The loss of this bit makes the other one continue to spin while the
90 * thread is working alone.
91 */
92}
93
94/* Cancels the effect of thread_isolate() by releasing the current thread's bit
95 * in threads_want_rdv_mask and by marking this thread as harmless until the
96 * last worker finishes.
97 */
98void thread_release()
99{
Olivier Houchardb23a61f2019-03-08 18:51:17 +0100100 _HA_ATOMIC_AND(&threads_want_rdv_mask, ~tid_bit);
Willy Tarreau31cba0d2019-06-09 08:44:19 +0200101 while (threads_want_rdv_mask & all_threads_mask) {
102 _HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
103 while (threads_want_rdv_mask & all_threads_mask)
104 ha_thread_relax();
105 HA_ATOMIC_AND(&threads_harmless_mask, ~tid_bit);
106 }
Willy Tarreau60b639c2018-08-02 10:16:17 +0200107}
Christopher Faulet339fff82017-10-19 11:59:15 +0200108
Willy Tarreau9a1f5732019-06-09 12:20:02 +0200109/* Cancels the effect of thread_isolate() by releasing the current thread's bit
110 * in threads_want_rdv_mask and by marking this thread as harmless until the
111 * last worker finishes. The difference with thread_release() is that this one
112 * will not leave the function before others are notified to do the same, so it
113 * guarantees that the current thread will not pass through a subsequent call
114 * to thread_isolate() before others finish.
115 */
116void thread_sync_release()
117{
118 _HA_ATOMIC_OR(&threads_sync_mask, tid_bit);
119 __ha_barrier_atomic_store();
120 _HA_ATOMIC_AND(&threads_want_rdv_mask, ~tid_bit);
121
122 while (threads_want_rdv_mask & all_threads_mask) {
123 _HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
124 while (threads_want_rdv_mask & all_threads_mask)
125 ha_thread_relax();
126 HA_ATOMIC_AND(&threads_harmless_mask, ~tid_bit);
127 }
128
129 /* the current thread is not harmless anymore, thread_isolate()
130 * is forced to wait till all waiters finish.
131 */
132 _HA_ATOMIC_AND(&threads_sync_mask, ~tid_bit);
133 while (threads_sync_mask & all_threads_mask)
134 ha_thread_relax();
135}
136
Willy Tarreau2beaaf72019-05-22 08:43:34 +0200137/* send signal <sig> to thread <thr> */
138void ha_tkill(unsigned int thr, int sig)
139{
David Carliera92c5ce2019-09-13 05:03:12 +0100140 pthread_kill(ha_thread_info[thr].pthread, sig);
Willy Tarreau2beaaf72019-05-22 08:43:34 +0200141}
142
143/* send signal <sig> to all threads. The calling thread is signaled last in
144 * order to allow all threads to synchronize in the handler.
145 */
146void ha_tkillall(int sig)
147{
148 unsigned int thr;
149
150 for (thr = 0; thr < global.nbthread; thr++) {
151 if (!(all_threads_mask & (1UL << thr)))
152 continue;
153 if (thr == tid)
154 continue;
David Carliera92c5ce2019-09-13 05:03:12 +0100155 pthread_kill(ha_thread_info[thr].pthread, sig);
Willy Tarreau2beaaf72019-05-22 08:43:34 +0200156 }
157 raise(sig);
158}
159
Willy Tarreau3d184982020-10-18 10:20:59 +0200160/* these calls are used as callbacks at init time when debugging is on */
Willy Tarreaua8ae77d2018-11-25 19:28:23 +0100161void ha_spin_init(HA_SPINLOCK_T *l)
162{
163 HA_SPIN_INIT(l);
164}
165
Willy Tarreau3d184982020-10-18 10:20:59 +0200166/* these calls are used as callbacks at init time when debugging is on */
Willy Tarreaua8ae77d2018-11-25 19:28:23 +0100167void ha_rwlock_init(HA_RWLOCK_T *l)
168{
169 HA_RWLOCK_INIT(l);
170}
171
Willy Tarreau149ab772019-01-26 14:27:06 +0100172/* returns the number of CPUs the current process is enabled to run on */
173static int thread_cpus_enabled()
174{
175 int ret = 1;
176
177#ifdef USE_CPU_AFFINITY
178#if defined(__linux__) && defined(CPU_COUNT)
179 cpu_set_t mask;
180
181 if (sched_getaffinity(0, sizeof(mask), &mask) == 0)
182 ret = CPU_COUNT(&mask);
Olivier Houchard46453d32019-04-11 00:06:47 +0200183#elif defined(__FreeBSD__) && defined(USE_CPU_AFFINITY)
184 cpuset_t cpuset;
185 if (cpuset_getaffinity(CPU_LEVEL_CPUSET, CPU_WHICH_PID, -1,
186 sizeof(cpuset), &cpuset) == 0)
187 ret = CPU_COUNT(&cpuset);
David CARLIER6a906012021-01-15 08:09:56 +0000188#elif defined(__APPLE__)
189 ret = (int)sysconf(_SC_NPROCESSORS_ONLN);
Willy Tarreau149ab772019-01-26 14:27:06 +0100190#endif
191#endif
192 ret = MAX(ret, 1);
193 ret = MIN(ret, MAX_THREADS);
194 return ret;
195}
196
Amaury Denoyelle4c9efde2021-03-31 16:57:39 +0200197/* Returns 1 if the cpu set is currently restricted for the process else 0.
198 * Currently only implemented for the Linux platform.
199 */
200int thread_cpu_mask_forced()
201{
202#if defined(__linux__)
203 const int cpus_avail = sysconf(_SC_NPROCESSORS_ONLN);
204 return cpus_avail != thread_cpus_enabled();
205#else
206 return 0;
207#endif
208}
209
Willy Tarreauf734ebf2020-09-09 17:07:54 +0200210/* Depending on the platform and how libpthread was built, pthread_exit() may
211 * involve some code in libgcc_s that would be loaded on exit for the first
212 * time, causing aborts if the process is chrooted. It's harmless bit very
213 * dirty. There isn't much we can do to make sure libgcc_s is loaded only if
214 * needed, so what we do here is that during early boot we create a dummy
215 * thread that immediately exits. This will lead to libgcc_s being loaded
216 * during boot on the platforms where it's required.
217 */
218static void *dummy_thread_function(void *data)
219{
220 pthread_exit(NULL);
221 return NULL;
222}
223
224static inline void preload_libgcc_s(void)
225{
226 pthread_t dummy_thread;
227 pthread_create(&dummy_thread, NULL, dummy_thread_function, NULL);
228 pthread_join(dummy_thread, NULL);
229}
230
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200231__attribute__((constructor))
Willy Tarreau3f567e42020-05-28 15:29:19 +0200232static void __thread_init(void)
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200233{
Willy Tarreauf5809cd2019-01-26 13:35:03 +0100234 char *ptr = NULL;
235
236 if (MAX_THREADS < 1 || MAX_THREADS > LONGBITS) {
237 ha_alert("MAX_THREADS value must be between 1 and %d inclusive; "
238 "HAProxy was built with value %d, please fix it and rebuild.\n",
239 LONGBITS, MAX_THREADS);
240 exit(1);
241 }
Willy Tarreau149ab772019-01-26 14:27:06 +0100242
Willy Tarreauf734ebf2020-09-09 17:07:54 +0200243 preload_libgcc_s();
Willy Tarreau77b98222020-09-02 08:04:35 +0200244
Willy Tarreau149ab772019-01-26 14:27:06 +0100245 thread_cpus_enabled_at_boot = thread_cpus_enabled();
246
247 memprintf(&ptr, "Built with multi-threading support (MAX_THREADS=%d, default=%d).",
248 MAX_THREADS, thread_cpus_enabled_at_boot);
Willy Tarreauf5809cd2019-01-26 13:35:03 +0100249 hap_register_build_opts(ptr, 1);
250
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200251#if defined(DEBUG_THREAD) || defined(DEBUG_FULL)
252 memset(lock_stats, 0, sizeof(lock_stats));
253#endif
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200254}
255
Willy Tarreau8459f252018-12-15 16:48:14 +0100256#else
257
258REGISTER_BUILD_OPTS("Built without multi-threading support (USE_THREAD not set).");
259
Willy Tarreau0ccd3222018-07-30 10:34:35 +0200260#endif // USE_THREAD
261
262
263/* Parse the number of threads in argument <arg>, returns it and adjusts a few
264 * internal variables accordingly, or fails and returns zero with an error
265 * reason in <errmsg>. May be called multiple times while parsing.
266 */
267int parse_nbthread(const char *arg, char **err)
268{
269 long nbthread;
270 char *errptr;
271
272 nbthread = strtol(arg, &errptr, 10);
273 if (!*arg || *errptr) {
274 memprintf(err, "passed a missing or unparsable integer value in '%s'", arg);
275 return 0;
276 }
277
278#ifndef USE_THREAD
279 if (nbthread != 1) {
280 memprintf(err, "specified with a value other than 1 while HAProxy is not compiled with threads support. Please check build options for USE_THREAD");
281 return 0;
282 }
283#else
284 if (nbthread < 1 || nbthread > MAX_THREADS) {
285 memprintf(err, "value must be between 1 and %d (was %ld)", MAX_THREADS, nbthread);
286 return 0;
287 }
288
Willy Tarreaufc647362019-02-02 17:05:03 +0100289 all_threads_mask = nbits(nbthread);
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200290#endif
Willy Tarreau0ccd3222018-07-30 10:34:35 +0200291 return nbthread;
292}