blob: adaf6f4613c9d97ec777aacece938af0e7d5ba2a [file] [log] [blame]
Willy Tarreau2bfefdb2019-05-03 13:52:18 +02001/*
2 * Thread lockup detection
3 *
4 * Copyright 2000-2019 Willy Tarreau <willy@haproxy.org>.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11
12#include <signal.h>
13#include <time.h>
14
15#include <common/config.h>
16#include <common/debug.h>
17#include <common/hathreads.h>
18#include <common/initcall.h>
19#include <common/standard.h>
20#include <types/global.h>
21#include <proto/log.h>
22
23
24/*
25 * It relies on timer_create() and timer_settime() which are only available in
26 * this case.
27 */
Willy Tarreaubc1b8202019-05-23 10:20:55 +020028#if defined(USE_THREAD) && defined(USE_RT) && (_POSIX_TIMERS > 0) && defined(_POSIX_THREAD_CPUTIME)
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020029
30/* We'll deliver SIGALRM when we've run out of CPU as it's not intercepted by
31 * gdb by default.
32 */
33#define WDTSIG SIGALRM
34
35/* Setup (or ping) the watchdog timer for thread <thr>. Returns non-zero on
36 * success, zero on failure. It interrupts once per second of CPU time. It
37 * happens that timers based on the CPU time are not automatically re-armed
38 * so we only use the value and leave the interval unset.
39 */
40int wdt_ping(int thr)
41{
42 struct itimerspec its;
43
44 its.it_value.tv_sec = 1; its.it_value.tv_nsec = 0;
45 its.it_interval.tv_sec = 0; its.it_interval.tv_nsec = 0;
46 return timer_settime(thread_info[thr].wd_timer, 0, &its, NULL) == 0;
47}
48
49/* This is the WDTSIG signal handler */
50void wdt_handler(int sig, siginfo_t *si, void *arg)
51{
52 unsigned long long n, p;
53 int thr;
54
55 switch (si->si_code) {
56 case SI_TIMER:
57 /* A thread's timer fired, the thread ID is in si_int. We have
58 * no guarantee that the thread handling this signal is in any
59 * way related to the one triggering it, so we need to retrieve
60 * the thread number from there. Note: this thread might
61 * continue to execute in parallel.
62 */
Willy Tarreau02255b22019-05-23 08:36:29 +020063 thr = si->si_value.sival_int;
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020064
65 /* cannot happen unless an unknown timer tries to play with our
66 * nerves. Let's die for now if this happens.
67 */
68 if (thr < 0 || thr >= global.nbthread)
69 break;
70
71 p = thread_info[thr].prev_cpu_time;
72 n = now_cpu_time_thread(&thread_info[thr]);
73
74 /* not yet reached the deadline of 1 sec */
75 if (n - p < 1000000000UL)
76 goto update_and_leave;
77
Willy Tarreau445b2b72019-07-31 19:20:39 +020078 if ((threads_harmless_mask|sleeping_thread_mask|threads_to_dump) & (1UL << thr)) {
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020079 /* This thread is currently doing exactly nothing
80 * waiting in the poll loop (unlikely but possible),
81 * waiting for all other threads to join the rendez-vous
82 * point (common), or waiting for another thread to
83 * finish an isolated operation (unlikely but possible).
84 */
85 goto update_and_leave;
86 }
87
88 /* So the thread indeed appears locked up. In order to be
89 * certain that we're not witnessing an exceptional spike of
90 * CPU usage due to a configuration issue (like running tens
91 * of thousands of tasks in a single loop), we'll check if the
92 * scheduler is still alive by setting the TI_FL_STUCK flag
93 * that the scheduler clears when switching to the next task.
94 * If it's already set, then it's our second call with no
95 * progress and the thread is dead.
96 */
97 if (!(thread_info[thr].flags & TI_FL_STUCK)) {
98 _HA_ATOMIC_OR(&thread_info[thr].flags, TI_FL_STUCK);
99 goto update_and_leave;
100 }
101
102 /* No doubt now, there's no hop to recover, die loudly! */
103 break;
104
105 case SI_TKILL:
106 /* we got a pthread_kill, stop on it */
107 thr = tid;
108 break;
109
110 default:
111 /* unhandled other conditions */
112 return;
113 }
114
115 /* By default we terminate. If we're not on the victim thread, better
116 * bounce the signal there so that we produce a cleaner stack trace
117 * with the other thread interrupted exactly where it was running and
118 * the current one not involved in this.
119 */
120 if (thr != tid)
121 pthread_kill(thread_info[thr].pthread, sig);
122 else
123 ha_panic();
124 return;
125
126 update_and_leave:
127 wdt_ping(thr);
128}
129
130int init_wdt_per_thread()
131{
132 struct sigevent sev;
133 sigset_t set;
134
135 /* unblock the WDTSIG signal we intend to use */
136 sigemptyset(&set);
137 sigaddset(&set, WDTSIG);
138 ha_sigmask(SIG_UNBLOCK, &set, NULL);
139
140 /* this timer will signal WDTSIG when it fires, with tid in the si_int
141 * field (important since any thread will receive the signal).
142 */
143 sev.sigev_notify = SIGEV_SIGNAL;
144 sev.sigev_signo = WDTSIG;
145 sev.sigev_value.sival_int = tid;
146 if (timer_create(ti->clock_id, &sev, &ti->wd_timer) == -1)
147 goto fail1;
148
149 if (!wdt_ping(tid))
150 goto fail2;
151
152 return 1;
153
154 fail2:
155 timer_delete(ti->wd_timer);
156 fail1:
157 ti->wd_timer = TIMER_INVALID;
158 ha_warning("Failed to setup watchdog timer for thread %u, disabling lockup detection.\n", tid);
Willy Tarreau6a44bde2020-03-04 10:46:13 +0100159 return 1;
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200160}
161
162void deinit_wdt_per_thread()
163{
164 if (ti->wd_timer != TIMER_INVALID)
165 timer_delete(ti->wd_timer);
166}
167
168/* registers the watchdog signal handler and returns 0. This sets up the signal
169 * handler for WDTSIG, so it must be called once per process.
170 */
171int init_wdt()
172{
173 struct sigaction sa;
174
175 sa.sa_handler = NULL;
176 sa.sa_sigaction = wdt_handler;
177 sigemptyset(&sa.sa_mask);
178 sa.sa_flags = SA_SIGINFO;
179 sigaction(WDTSIG, &sa, NULL);
180 return 0;
181}
182
183REGISTER_POST_CHECK(init_wdt);
184REGISTER_PER_THREAD_INIT(init_wdt_per_thread);
185REGISTER_PER_THREAD_DEINIT(deinit_wdt_per_thread);
186#endif