blob: 44b35402abbbbcb7ce5598c363dbdfbdcc12fb3d [file] [log] [blame]
Willy Tarreau2bfefdb2019-05-03 13:52:18 +02001/*
2 * Thread lockup detection
3 *
4 * Copyright 2000-2019 Willy Tarreau <willy@haproxy.org>.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11
12#include <signal.h>
13#include <time.h>
14
15#include <common/config.h>
16#include <common/debug.h>
17#include <common/hathreads.h>
18#include <common/initcall.h>
19#include <common/standard.h>
20#include <types/global.h>
21#include <proto/log.h>
22
23
24/*
25 * It relies on timer_create() and timer_settime() which are only available in
26 * this case.
27 */
Willy Tarreaue58114e2020-03-04 10:53:07 +010028#if defined(USE_RT) && (_POSIX_TIMERS > 0) && defined(_POSIX_THREAD_CPUTIME)
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020029
30/* We'll deliver SIGALRM when we've run out of CPU as it's not intercepted by
31 * gdb by default.
32 */
33#define WDTSIG SIGALRM
34
35/* Setup (or ping) the watchdog timer for thread <thr>. Returns non-zero on
36 * success, zero on failure. It interrupts once per second of CPU time. It
37 * happens that timers based on the CPU time are not automatically re-armed
38 * so we only use the value and leave the interval unset.
39 */
40int wdt_ping(int thr)
41{
42 struct itimerspec its;
43
44 its.it_value.tv_sec = 1; its.it_value.tv_nsec = 0;
45 its.it_interval.tv_sec = 0; its.it_interval.tv_nsec = 0;
David Carliera92c5ce2019-09-13 05:03:12 +010046 return timer_settime(ha_thread_info[thr].wd_timer, 0, &its, NULL) == 0;
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020047}
48
49/* This is the WDTSIG signal handler */
50void wdt_handler(int sig, siginfo_t *si, void *arg)
51{
52 unsigned long long n, p;
53 int thr;
54
55 switch (si->si_code) {
56 case SI_TIMER:
57 /* A thread's timer fired, the thread ID is in si_int. We have
58 * no guarantee that the thread handling this signal is in any
59 * way related to the one triggering it, so we need to retrieve
60 * the thread number from there. Note: this thread might
61 * continue to execute in parallel.
62 */
Willy Tarreau02255b22019-05-23 08:36:29 +020063 thr = si->si_value.sival_int;
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020064
65 /* cannot happen unless an unknown timer tries to play with our
66 * nerves. Let's die for now if this happens.
67 */
68 if (thr < 0 || thr >= global.nbthread)
69 break;
70
David Carliera92c5ce2019-09-13 05:03:12 +010071 p = ha_thread_info[thr].prev_cpu_time;
72 n = now_cpu_time_thread(&ha_thread_info[thr]);
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020073
74 /* not yet reached the deadline of 1 sec */
75 if (n - p < 1000000000UL)
76 goto update_and_leave;
77
Willy Tarreaua37cb182019-07-31 19:20:39 +020078 if ((threads_harmless_mask|sleeping_thread_mask|threads_to_dump) & (1UL << thr)) {
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020079 /* This thread is currently doing exactly nothing
80 * waiting in the poll loop (unlikely but possible),
81 * waiting for all other threads to join the rendez-vous
82 * point (common), or waiting for another thread to
83 * finish an isolated operation (unlikely but possible).
84 */
85 goto update_and_leave;
86 }
87
88 /* So the thread indeed appears locked up. In order to be
89 * certain that we're not witnessing an exceptional spike of
90 * CPU usage due to a configuration issue (like running tens
91 * of thousands of tasks in a single loop), we'll check if the
92 * scheduler is still alive by setting the TI_FL_STUCK flag
93 * that the scheduler clears when switching to the next task.
94 * If it's already set, then it's our second call with no
95 * progress and the thread is dead.
96 */
David Carliera92c5ce2019-09-13 05:03:12 +010097 if (!(ha_thread_info[thr].flags & TI_FL_STUCK)) {
98 _HA_ATOMIC_OR(&ha_thread_info[thr].flags, TI_FL_STUCK);
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020099 goto update_and_leave;
100 }
101
102 /* No doubt now, there's no hop to recover, die loudly! */
103 break;
Willy Tarreau06278152020-03-10 09:26:17 +0100104#ifdef USE_THREAD
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200105 case SI_TKILL:
106 /* we got a pthread_kill, stop on it */
107 thr = tid;
108 break;
Willy Tarreau06278152020-03-10 09:26:17 +0100109#endif
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200110 default:
111 /* unhandled other conditions */
112 return;
113 }
114
115 /* By default we terminate. If we're not on the victim thread, better
116 * bounce the signal there so that we produce a cleaner stack trace
117 * with the other thread interrupted exactly where it was running and
118 * the current one not involved in this.
119 */
Willy Tarreaue58114e2020-03-04 10:53:07 +0100120#ifdef USE_THREAD
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200121 if (thr != tid)
David Carliera92c5ce2019-09-13 05:03:12 +0100122 pthread_kill(ha_thread_info[thr].pthread, sig);
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200123 else
Willy Tarreaue58114e2020-03-04 10:53:07 +0100124#endif
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200125 ha_panic();
126 return;
127
128 update_and_leave:
129 wdt_ping(thr);
130}
131
132int init_wdt_per_thread()
133{
Willy Tarreauc1563e52020-02-26 14:03:05 +0100134 struct sigevent sev = { };
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200135 sigset_t set;
136
137 /* unblock the WDTSIG signal we intend to use */
138 sigemptyset(&set);
139 sigaddset(&set, WDTSIG);
140 ha_sigmask(SIG_UNBLOCK, &set, NULL);
141
142 /* this timer will signal WDTSIG when it fires, with tid in the si_int
143 * field (important since any thread will receive the signal).
144 */
145 sev.sigev_notify = SIGEV_SIGNAL;
146 sev.sigev_signo = WDTSIG;
147 sev.sigev_value.sival_int = tid;
Willy Tarreaud6f19662020-03-04 10:48:18 +0100148 if (timer_create(ti->clock_id, &sev, &ti->wd_timer) == -1 &&
149 timer_create(CLOCK_REALTIME, &sev, &ti->wd_timer) == -1)
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200150 goto fail1;
151
152 if (!wdt_ping(tid))
153 goto fail2;
154
155 return 1;
156
157 fail2:
158 timer_delete(ti->wd_timer);
159 fail1:
160 ti->wd_timer = TIMER_INVALID;
161 ha_warning("Failed to setup watchdog timer for thread %u, disabling lockup detection.\n", tid);
Willy Tarreau7259fa22020-03-04 10:46:13 +0100162 return 1;
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200163}
164
165void deinit_wdt_per_thread()
166{
167 if (ti->wd_timer != TIMER_INVALID)
168 timer_delete(ti->wd_timer);
169}
170
171/* registers the watchdog signal handler and returns 0. This sets up the signal
172 * handler for WDTSIG, so it must be called once per process.
173 */
174int init_wdt()
175{
176 struct sigaction sa;
177
178 sa.sa_handler = NULL;
179 sa.sa_sigaction = wdt_handler;
180 sigemptyset(&sa.sa_mask);
181 sa.sa_flags = SA_SIGINFO;
182 sigaction(WDTSIG, &sa, NULL);
183 return 0;
184}
185
186REGISTER_POST_CHECK(init_wdt);
187REGISTER_PER_THREAD_INIT(init_wdt_per_thread);
188REGISTER_PER_THREAD_DEINIT(deinit_wdt_per_thread);
189#endif