blob: 1248bebbb88cea6f4e0ff73c7a187c6767bdaf42 [file] [log] [blame]
Willy Tarreau2bfefdb2019-05-03 13:52:18 +02001/*
2 * Thread lockup detection
3 *
4 * Copyright 2000-2019 Willy Tarreau <willy@haproxy.org>.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11
12#include <signal.h>
13#include <time.h>
14
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020015#include <haproxy/api.h>
Willy Tarreau2a83d602020-05-27 16:58:08 +020016#include <haproxy/debug.h>
Willy Tarreau3f567e42020-05-28 15:29:19 +020017#include <haproxy/thread.h>
Willy Tarreau48fbcae2020-06-03 18:09:46 +020018#include <haproxy/tools.h>
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020019#include <types/global.h>
Olivier Houchardde01ea92020-03-18 13:07:19 +010020#include <types/signal.h>
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020021#include <proto/log.h>
22
23
24/*
25 * It relies on timer_create() and timer_settime() which are only available in
26 * this case.
27 */
Willy Tarreaue58114e2020-03-04 10:53:07 +010028#if defined(USE_RT) && (_POSIX_TIMERS > 0) && defined(_POSIX_THREAD_CPUTIME)
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020029
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020030/* Setup (or ping) the watchdog timer for thread <thr>. Returns non-zero on
31 * success, zero on failure. It interrupts once per second of CPU time. It
32 * happens that timers based on the CPU time are not automatically re-armed
33 * so we only use the value and leave the interval unset.
34 */
35int wdt_ping(int thr)
36{
37 struct itimerspec its;
38
39 its.it_value.tv_sec = 1; its.it_value.tv_nsec = 0;
40 its.it_interval.tv_sec = 0; its.it_interval.tv_nsec = 0;
David Carliera92c5ce2019-09-13 05:03:12 +010041 return timer_settime(ha_thread_info[thr].wd_timer, 0, &its, NULL) == 0;
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020042}
43
44/* This is the WDTSIG signal handler */
45void wdt_handler(int sig, siginfo_t *si, void *arg)
46{
47 unsigned long long n, p;
48 int thr;
49
50 switch (si->si_code) {
51 case SI_TIMER:
52 /* A thread's timer fired, the thread ID is in si_int. We have
53 * no guarantee that the thread handling this signal is in any
54 * way related to the one triggering it, so we need to retrieve
55 * the thread number from there. Note: this thread might
56 * continue to execute in parallel.
57 */
Willy Tarreau02255b22019-05-23 08:36:29 +020058 thr = si->si_value.sival_int;
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020059
60 /* cannot happen unless an unknown timer tries to play with our
61 * nerves. Let's die for now if this happens.
62 */
63 if (thr < 0 || thr >= global.nbthread)
64 break;
65
David Carliera92c5ce2019-09-13 05:03:12 +010066 p = ha_thread_info[thr].prev_cpu_time;
67 n = now_cpu_time_thread(&ha_thread_info[thr]);
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020068
69 /* not yet reached the deadline of 1 sec */
70 if (n - p < 1000000000UL)
71 goto update_and_leave;
72
Willy Tarreaua37cb182019-07-31 19:20:39 +020073 if ((threads_harmless_mask|sleeping_thread_mask|threads_to_dump) & (1UL << thr)) {
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020074 /* This thread is currently doing exactly nothing
75 * waiting in the poll loop (unlikely but possible),
76 * waiting for all other threads to join the rendez-vous
77 * point (common), or waiting for another thread to
78 * finish an isolated operation (unlikely but possible).
79 */
80 goto update_and_leave;
81 }
82
83 /* So the thread indeed appears locked up. In order to be
84 * certain that we're not witnessing an exceptional spike of
85 * CPU usage due to a configuration issue (like running tens
86 * of thousands of tasks in a single loop), we'll check if the
87 * scheduler is still alive by setting the TI_FL_STUCK flag
88 * that the scheduler clears when switching to the next task.
89 * If it's already set, then it's our second call with no
90 * progress and the thread is dead.
91 */
David Carliera92c5ce2019-09-13 05:03:12 +010092 if (!(ha_thread_info[thr].flags & TI_FL_STUCK)) {
93 _HA_ATOMIC_OR(&ha_thread_info[thr].flags, TI_FL_STUCK);
Willy Tarreau2bfefdb2019-05-03 13:52:18 +020094 goto update_and_leave;
95 }
96
97 /* No doubt now, there's no hop to recover, die loudly! */
98 break;
Willy Tarreau06278152020-03-10 09:26:17 +010099#ifdef USE_THREAD
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200100 case SI_TKILL:
101 /* we got a pthread_kill, stop on it */
102 thr = tid;
103 break;
Willy Tarreau06278152020-03-10 09:26:17 +0100104#endif
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200105 default:
106 /* unhandled other conditions */
107 return;
108 }
109
110 /* By default we terminate. If we're not on the victim thread, better
111 * bounce the signal there so that we produce a cleaner stack trace
112 * with the other thread interrupted exactly where it was running and
113 * the current one not involved in this.
114 */
Willy Tarreaue58114e2020-03-04 10:53:07 +0100115#ifdef USE_THREAD
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200116 if (thr != tid)
David Carliera92c5ce2019-09-13 05:03:12 +0100117 pthread_kill(ha_thread_info[thr].pthread, sig);
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200118 else
Willy Tarreaue58114e2020-03-04 10:53:07 +0100119#endif
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200120 ha_panic();
121 return;
122
123 update_and_leave:
124 wdt_ping(thr);
125}
126
127int init_wdt_per_thread()
128{
Willy Tarreauc1563e52020-02-26 14:03:05 +0100129 struct sigevent sev = { };
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200130 sigset_t set;
131
132 /* unblock the WDTSIG signal we intend to use */
133 sigemptyset(&set);
134 sigaddset(&set, WDTSIG);
135 ha_sigmask(SIG_UNBLOCK, &set, NULL);
136
137 /* this timer will signal WDTSIG when it fires, with tid in the si_int
138 * field (important since any thread will receive the signal).
139 */
140 sev.sigev_notify = SIGEV_SIGNAL;
141 sev.sigev_signo = WDTSIG;
142 sev.sigev_value.sival_int = tid;
Willy Tarreaud6f19662020-03-04 10:48:18 +0100143 if (timer_create(ti->clock_id, &sev, &ti->wd_timer) == -1 &&
144 timer_create(CLOCK_REALTIME, &sev, &ti->wd_timer) == -1)
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200145 goto fail1;
146
147 if (!wdt_ping(tid))
148 goto fail2;
149
150 return 1;
151
152 fail2:
153 timer_delete(ti->wd_timer);
154 fail1:
155 ti->wd_timer = TIMER_INVALID;
156 ha_warning("Failed to setup watchdog timer for thread %u, disabling lockup detection.\n", tid);
Willy Tarreau7259fa22020-03-04 10:46:13 +0100157 return 1;
Willy Tarreau2bfefdb2019-05-03 13:52:18 +0200158}
159
160void deinit_wdt_per_thread()
161{
162 if (ti->wd_timer != TIMER_INVALID)
163 timer_delete(ti->wd_timer);
164}
165
166/* registers the watchdog signal handler and returns 0. This sets up the signal
167 * handler for WDTSIG, so it must be called once per process.
168 */
169int init_wdt()
170{
171 struct sigaction sa;
172
173 sa.sa_handler = NULL;
174 sa.sa_sigaction = wdt_handler;
175 sigemptyset(&sa.sa_mask);
176 sa.sa_flags = SA_SIGINFO;
177 sigaction(WDTSIG, &sa, NULL);
178 return 0;
179}
180
181REGISTER_POST_CHECK(init_wdt);
182REGISTER_PER_THREAD_INIT(init_wdt_per_thread);
183REGISTER_PER_THREAD_DEINIT(deinit_wdt_per_thread);
184#endif