blob: 1826f9201fc289efb7325af4bf5315c28d33096a [file] [log] [blame]
Christopher Faulet1a2b56e2017-10-12 16:09:09 +02001/*
2 * functions about threads.
3 *
4 * Copyright (C) 2017 Christopher Fauet - cfaulet@haproxy.com
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
Willy Tarreau149ab772019-01-26 14:27:06 +010013#define _GNU_SOURCE
Christopher Faulet339fff82017-10-19 11:59:15 +020014#include <unistd.h>
Willy Tarreau0ccd3222018-07-30 10:34:35 +020015#include <stdlib.h>
Christopher Faulet339fff82017-10-19 11:59:15 +020016#include <fcntl.h>
17
Willy Tarreau149ab772019-01-26 14:27:06 +010018#ifdef USE_CPU_AFFINITY
19#include <sched.h>
20#endif
21
Willy Tarreau04931492017-11-03 23:39:25 +010022#include <common/cfgparse.h>
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020023#include <common/hathreads.h>
Christopher Faulet339fff82017-10-19 11:59:15 +020024#include <common/standard.h>
Willy Tarreau80713382018-11-26 10:19:54 +010025#include <types/global.h>
Christopher Faulet339fff82017-10-19 11:59:15 +020026#include <proto/fd.h>
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020027
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020028
29#ifdef USE_THREAD
30
Willy Tarreau60b639c2018-08-02 10:16:17 +020031volatile unsigned long threads_want_rdv_mask = 0;
32volatile unsigned long threads_harmless_mask = 0;
Willy Tarreau0ccd3222018-07-30 10:34:35 +020033volatile unsigned long all_threads_mask = 1; // nbthread 1 assumed by default
Willy Tarreau0c026f42018-08-01 19:12:20 +020034THREAD_LOCAL unsigned int tid = 0;
35THREAD_LOCAL unsigned long tid_bit = (1UL << 0);
Willy Tarreau149ab772019-01-26 14:27:06 +010036int thread_cpus_enabled_at_boot = 1;
Willy Tarreau0c026f42018-08-01 19:12:20 +020037
Christopher Faulet339fff82017-10-19 11:59:15 +020038
Christopher Faulet1a2b56e2017-10-12 16:09:09 +020039#if defined(DEBUG_THREAD) || defined(DEBUG_FULL)
40struct lock_stat lock_stats[LOCK_LABELS];
41#endif
42
Willy Tarreau60b639c2018-08-02 10:16:17 +020043/* Marks the thread as harmless until the last thread using the rendez-vous
44 * point quits. Given that we can wait for a long time, sched_yield() is used
45 * when available to offer the CPU resources to competing threads if needed.
46 */
47void thread_harmless_till_end()
48{
Olivier Houchardb23a61f2019-03-08 18:51:17 +010049 _HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
Willy Tarreau60b639c2018-08-02 10:16:17 +020050 while (threads_want_rdv_mask & all_threads_mask) {
51#if _POSIX_PRIORITY_SCHEDULING
52 sched_yield();
53#else
54 pl_cpu_relax();
55#endif
56 }
57}
58
59/* Isolates the current thread : request the ability to work while all other
60 * threads are harmless. Only returns once all of them are harmless, with the
61 * current thread's bit in threads_harmless_mask cleared. Needs to be completed
62 * using thread_release().
63 */
64void thread_isolate()
65{
66 unsigned long old;
67
Olivier Houchardb23a61f2019-03-08 18:51:17 +010068 _HA_ATOMIC_OR(&threads_harmless_mask, tid_bit);
69 __ha_barrier_atomic_store();
70 _HA_ATOMIC_OR(&threads_want_rdv_mask, tid_bit);
Willy Tarreau60b639c2018-08-02 10:16:17 +020071
72 /* wait for all threads to become harmless */
73 old = threads_harmless_mask;
74 while (1) {
75 if (unlikely((old & all_threads_mask) != all_threads_mask))
76 old = threads_harmless_mask;
Olivier Houchardb23a61f2019-03-08 18:51:17 +010077 else if (_HA_ATOMIC_CAS(&threads_harmless_mask, &old, old & ~tid_bit))
Willy Tarreau60b639c2018-08-02 10:16:17 +020078 break;
79
80#if _POSIX_PRIORITY_SCHEDULING
81 sched_yield();
82#else
83 pl_cpu_relax();
84#endif
85 }
86 /* one thread gets released at a time here, with its harmess bit off.
87 * The loss of this bit makes the other one continue to spin while the
88 * thread is working alone.
89 */
90}
91
92/* Cancels the effect of thread_isolate() by releasing the current thread's bit
93 * in threads_want_rdv_mask and by marking this thread as harmless until the
94 * last worker finishes.
95 */
96void thread_release()
97{
Olivier Houchardb23a61f2019-03-08 18:51:17 +010098 _HA_ATOMIC_AND(&threads_want_rdv_mask, ~tid_bit);
Willy Tarreaua9c02522018-10-16 16:11:56 +020099 thread_harmless_end();
Willy Tarreau60b639c2018-08-02 10:16:17 +0200100}
Christopher Faulet339fff82017-10-19 11:59:15 +0200101
Willy Tarreaua8ae77d2018-11-25 19:28:23 +0100102/* these calls are used as callbacks at init time */
103void ha_spin_init(HA_SPINLOCK_T *l)
104{
105 HA_SPIN_INIT(l);
106}
107
108/* these calls are used as callbacks at init time */
109void ha_rwlock_init(HA_RWLOCK_T *l)
110{
111 HA_RWLOCK_INIT(l);
112}
113
Willy Tarreau149ab772019-01-26 14:27:06 +0100114/* returns the number of CPUs the current process is enabled to run on */
115static int thread_cpus_enabled()
116{
117 int ret = 1;
118
119#ifdef USE_CPU_AFFINITY
120#if defined(__linux__) && defined(CPU_COUNT)
121 cpu_set_t mask;
122
123 if (sched_getaffinity(0, sizeof(mask), &mask) == 0)
124 ret = CPU_COUNT(&mask);
125#endif
126#endif
127 ret = MAX(ret, 1);
128 ret = MIN(ret, MAX_THREADS);
129 return ret;
130}
131
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200132__attribute__((constructor))
133static void __hathreads_init(void)
134{
Willy Tarreauf5809cd2019-01-26 13:35:03 +0100135 char *ptr = NULL;
136
137 if (MAX_THREADS < 1 || MAX_THREADS > LONGBITS) {
138 ha_alert("MAX_THREADS value must be between 1 and %d inclusive; "
139 "HAProxy was built with value %d, please fix it and rebuild.\n",
140 LONGBITS, MAX_THREADS);
141 exit(1);
142 }
Willy Tarreau149ab772019-01-26 14:27:06 +0100143
144 thread_cpus_enabled_at_boot = thread_cpus_enabled();
145
146 memprintf(&ptr, "Built with multi-threading support (MAX_THREADS=%d, default=%d).",
147 MAX_THREADS, thread_cpus_enabled_at_boot);
Willy Tarreauf5809cd2019-01-26 13:35:03 +0100148 hap_register_build_opts(ptr, 1);
149
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200150#if defined(DEBUG_THREAD) || defined(DEBUG_FULL)
151 memset(lock_stats, 0, sizeof(lock_stats));
152#endif
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200153}
154
Willy Tarreau8459f252018-12-15 16:48:14 +0100155#else
156
157REGISTER_BUILD_OPTS("Built without multi-threading support (USE_THREAD not set).");
158
Willy Tarreau0ccd3222018-07-30 10:34:35 +0200159#endif // USE_THREAD
160
161
162/* Parse the number of threads in argument <arg>, returns it and adjusts a few
163 * internal variables accordingly, or fails and returns zero with an error
164 * reason in <errmsg>. May be called multiple times while parsing.
165 */
166int parse_nbthread(const char *arg, char **err)
167{
168 long nbthread;
169 char *errptr;
170
171 nbthread = strtol(arg, &errptr, 10);
172 if (!*arg || *errptr) {
173 memprintf(err, "passed a missing or unparsable integer value in '%s'", arg);
174 return 0;
175 }
176
177#ifndef USE_THREAD
178 if (nbthread != 1) {
179 memprintf(err, "specified with a value other than 1 while HAProxy is not compiled with threads support. Please check build options for USE_THREAD");
180 return 0;
181 }
182#else
183 if (nbthread < 1 || nbthread > MAX_THREADS) {
184 memprintf(err, "value must be between 1 and %d (was %ld)", MAX_THREADS, nbthread);
185 return 0;
186 }
187
Willy Tarreaufc647362019-02-02 17:05:03 +0100188 all_threads_mask = nbits(nbthread);
Christopher Faulet1a2b56e2017-10-12 16:09:09 +0200189#endif
Willy Tarreau0ccd3222018-07-30 10:34:35 +0200190 return nbthread;
191}