blob: 474a6647f0bd94211e61f87028aa547e1c135cbb [file] [log] [blame]
Willy Tarreau4e2b6462019-05-16 17:44:30 +02001/*
2 * Process debugging functions.
3 *
4 * Copyright 2000-2019 Willy Tarreau <willy@haproxy.org>.
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
Willy Tarreauf5b4e062020-03-03 15:40:23 +010013
Willy Tarreau5be7c192022-01-24 20:26:09 +010014#include <errno.h>
Willy Tarreau368bff42019-12-06 17:18:28 +010015#include <fcntl.h>
Willy Tarreau4e2b6462019-05-16 17:44:30 +020016#include <signal.h>
17#include <time.h>
18#include <stdio.h>
Willy Tarreau6bdf3e92019-05-20 14:25:05 +020019#include <stdlib.h>
Willy Tarreauaeed4a82020-06-04 22:01:04 +020020#include <syslog.h>
Willy Tarreau5be7c192022-01-24 20:26:09 +010021#include <sys/stat.h>
Willy Tarreau368bff42019-12-06 17:18:28 +010022#include <sys/types.h>
23#include <sys/wait.h>
Willy Tarreau5be7c192022-01-24 20:26:09 +010024#include <unistd.h>
25#ifdef USE_EPOLL
26#include <sys/epoll.h>
27#endif
Willy Tarreau4e2b6462019-05-16 17:44:30 +020028
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020029#include <haproxy/api.h>
Christopher Faulet6b0a0fb2022-04-04 11:29:28 +020030#include <haproxy/applet.h>
Willy Tarreau8dabda72020-05-27 17:22:10 +020031#include <haproxy/buf.h>
Willy Tarreau83487a82020-06-04 20:19:54 +020032#include <haproxy/cli.h>
Willy Tarreau55542642021-10-08 09:33:24 +020033#include <haproxy/clock.h>
Willy Tarreau2a83d602020-05-27 16:58:08 +020034#include <haproxy/debug.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020035#include <haproxy/fd.h>
36#include <haproxy/global.h>
Willy Tarreau86416052020-06-04 09:20:54 +020037#include <haproxy/hlua.h>
Willy Tarreaub7fc4c42021-10-06 18:56:42 +020038#include <haproxy/http_ana.h>
Willy Tarreauaeed4a82020-06-04 22:01:04 +020039#include <haproxy/log.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020040#include <haproxy/net_helper.h>
Willy Tarreau5edca2f2022-05-27 09:25:10 +020041#include <haproxy/sc_strm.h>
Willy Tarreaucb086c62022-05-27 09:47:12 +020042#include <haproxy/stconn.h>
Willy Tarreaucea0e1b2020-06-04 17:25:40 +020043#include <haproxy/task.h>
Willy Tarreau3f567e42020-05-28 15:29:19 +020044#include <haproxy/thread.h>
Willy Tarreau55542642021-10-08 09:33:24 +020045#include <haproxy/time.h>
Willy Tarreau48fbcae2020-06-03 18:09:46 +020046#include <haproxy/tools.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020047#include <import/ist.h>
Willy Tarreau4e2b6462019-05-16 17:44:30 +020048
Willy Tarreau4e2b6462019-05-16 17:44:30 +020049
Willy Tarreauf0c86dd2022-07-15 13:14:03 +020050/* The dump state is made of:
51 * - num_thread on the lowest 15 bits
52 * - a SYNC flag on bit 15 (waiting for sync start)
53 * - number of participating threads on bits 16-30
54 * Initiating a dump consists in setting it to SYNC and incrementing the
55 * num_thread part when entering the function. The first thread periodically
56 * recounts active threads and compares it to the ready ones, and clears SYNC
57 * and sets the number of participants to the value found, which serves as a
58 * start signal. A thread finished dumping looks up the TID of the next active
59 * thread after it and writes it in the lowest part. If there's none, it sets
60 * the thread counter to the number of participants and resets that part,
61 * which serves as an end-of-dump signal. All threads decrement the num_thread
62 * part. Then all threads wait for the value to reach zero. Only used when
63 * USE_THREAD_DUMP is set.
Willy Tarreaua37cb182019-07-31 19:20:39 +020064 */
Willy Tarreauf0c86dd2022-07-15 13:14:03 +020065#define THREAD_DUMP_TMASK 0x00007FFFU
66#define THREAD_DUMP_FSYNC 0x00008000U
67#define THREAD_DUMP_PMASK 0x7FFF0000U
68
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +020069/* Points to a copy of the buffer where the dump functions should write, when
70 * non-null. It's only used by debuggers for core dump analysis.
71 */
72struct buffer *thread_dump_buffer = NULL;
Willy Tarreau9b013702019-10-24 18:18:02 +020073unsigned int debug_commands_issued = 0;
Willy Tarreaua37cb182019-07-31 19:20:39 +020074
Willy Tarreau123fc972021-01-22 13:52:41 +010075/* dumps a backtrace of the current thread that is appended to buffer <buf>.
76 * Lines are prefixed with the string <prefix> which may be empty (used for
77 * indenting). It is recommended to use this at a function's tail so that
Willy Tarreau2bfce7e2021-01-22 14:48:34 +010078 * the function does not appear in the call stack. The <dump> argument
79 * indicates what dump state to start from, and should usually be zero. It
80 * may be among the following values:
81 * - 0: search usual callers before step 1, or directly jump to 2
82 * - 1: skip usual callers before step 2
83 * - 2: dump until polling loop, scheduler, or main() (excluded)
84 * - 3: end
85 * - 4-7: like 0 but stops *after* main.
Willy Tarreau123fc972021-01-22 13:52:41 +010086 */
Willy Tarreau2bfce7e2021-01-22 14:48:34 +010087void ha_dump_backtrace(struct buffer *buf, const char *prefix, int dump)
Willy Tarreau123fc972021-01-22 13:52:41 +010088{
89 struct buffer bak;
90 char pfx2[100];
91 void *callers[100];
92 int j, nptrs;
93 const void *addr;
Willy Tarreau123fc972021-01-22 13:52:41 +010094
95 nptrs = my_backtrace(callers, sizeof(callers)/sizeof(*callers));
96 if (!nptrs)
97 return;
98
99 if (snprintf(pfx2, sizeof(pfx2), "%s| ", prefix) > sizeof(pfx2))
100 pfx2[0] = 0;
101
102 /* The call backtrace_symbols_fd(callers, nptrs, STDOUT_FILENO would
103 * produce similar output to the following:
104 */
105 chunk_appendf(buf, "%scall trace(%d):\n", prefix, nptrs);
Willy Tarreau2bfce7e2021-01-22 14:48:34 +0100106 for (j = 0; (j < nptrs || (dump & 3) < 2); j++) {
107 if (j == nptrs && !(dump & 3)) {
Willy Tarreau123fc972021-01-22 13:52:41 +0100108 /* we failed to spot the starting point of the
109 * dump, let's start over dumping everything we
110 * have.
111 */
Willy Tarreau2bfce7e2021-01-22 14:48:34 +0100112 dump += 2;
Willy Tarreau123fc972021-01-22 13:52:41 +0100113 j = 0;
114 }
115 bak = *buf;
116 dump_addr_and_bytes(buf, pfx2, callers[j], 8);
117 addr = resolve_sym_name(buf, ": ", callers[j]);
Willy Tarreau2bfce7e2021-01-22 14:48:34 +0100118 if ((dump & 3) == 0) {
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +0200119 /* dump not started, will start *after* ha_thread_dump_one(),
Willy Tarreaue69919d2023-05-04 19:19:04 +0200120 * ha_panic and ha_backtrace_to_stderr
Willy Tarreau123fc972021-01-22 13:52:41 +0100121 */
Willy Tarreaue69919d2023-05-04 19:19:04 +0200122 if (addr == ha_panic ||
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +0200123 addr == ha_backtrace_to_stderr || addr == ha_thread_dump_one)
Willy Tarreau2bfce7e2021-01-22 14:48:34 +0100124 dump++;
Willy Tarreau123fc972021-01-22 13:52:41 +0100125 *buf = bak;
126 continue;
127 }
128
Willy Tarreau2bfce7e2021-01-22 14:48:34 +0100129 if ((dump & 3) == 1) {
Willy Tarreau123fc972021-01-22 13:52:41 +0100130 /* starting */
Willy Tarreaue69919d2023-05-04 19:19:04 +0200131 if (addr == ha_panic ||
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +0200132 addr == ha_backtrace_to_stderr || addr == ha_thread_dump_one) {
Willy Tarreau123fc972021-01-22 13:52:41 +0100133 *buf = bak;
134 continue;
135 }
Willy Tarreau2bfce7e2021-01-22 14:48:34 +0100136 dump++;
Willy Tarreau123fc972021-01-22 13:52:41 +0100137 }
138
Willy Tarreau2bfce7e2021-01-22 14:48:34 +0100139 if ((dump & 3) == 2) {
140 /* still dumping */
141 if (dump == 6) {
142 /* we only stop *after* main and we must send the LF */
143 if (addr == main) {
144 j = nptrs;
145 dump++;
146 }
147 }
148 else if (addr == run_poll_loop || addr == main || addr == run_tasks_from_lists) {
149 dump++;
Willy Tarreau123fc972021-01-22 13:52:41 +0100150 *buf = bak;
151 break;
152 }
153 }
154 /* OK, line dumped */
155 chunk_appendf(buf, "\n");
156 }
157}
158
Willy Tarreaua8459b22021-01-22 14:12:27 +0100159/* dump a backtrace of current thread's stack to stderr. */
Willy Tarreau7831e022022-05-06 15:16:19 +0200160void ha_backtrace_to_stderr(void)
Willy Tarreaua8459b22021-01-22 14:12:27 +0100161{
162 char area[2048];
163 struct buffer b = b_make(area, sizeof(area), 0, 0);
164
Willy Tarreau2bfce7e2021-01-22 14:48:34 +0100165 ha_dump_backtrace(&b, " ", 4);
Willy Tarreaua8459b22021-01-22 14:12:27 +0100166 if (b.data)
Willy Tarreau2cbe2e72021-01-22 15:58:26 +0100167 DISGUISE(write(2, b.area, b.data));
Willy Tarreaua8459b22021-01-22 14:12:27 +0100168}
169
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +0200170/* Dumps to the thread's buffer some known information for the desired thread,
171 * and optionally extra info when it's safe to do so (current thread or
172 * isolated). The dump will be appended to the buffer, so the caller is
173 * responsible for preliminary initializing it. The <from_signal> argument will
174 * indicate if the function is called from the debug signal handler, indicating
175 * the thread was dumped upon request from another one, otherwise if the thread
176 * it the current one, a star ('*') will be displayed in front of the thread to
177 * indicate the requesting one. Any stuck thread is also prefixed with a '>'.
178 * The caller is responsible for atomically setting up the thread's dump buffer
179 * to point to a valid buffer with enough room. Output will be truncated if it
180 * does not fit. When the dump is complete, the dump buffer will be switched to
181 * (void*)0x1 that the caller must turn to 0x0 once the contents are collected.
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200182 */
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +0200183void ha_thread_dump_one(int thr, int from_signal)
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200184{
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +0200185 struct buffer *buf = HA_ATOMIC_LOAD(&ha_thread_ctx[thr].thread_dump_buffer);
Willy Tarreau95e6c992023-05-07 15:02:30 +0200186 unsigned long __maybe_unused thr_bit = ha_thread_info[thr].ltid_bit;
187 int __maybe_unused tgrp = ha_thread_info[thr].tgid;
Willy Tarreau45c38e22021-09-30 18:28:49 +0200188 unsigned long long p = ha_thread_ctx[thr].prev_cpu_time;
Willy Tarreau21694982021-10-08 15:09:17 +0200189 unsigned long long n = now_cpu_time_thread(thr);
Willy Tarreaua0b99532021-09-30 18:48:37 +0200190 int stuck = !!(ha_thread_ctx[thr].flags & TH_FL_STUCK);
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200191
Willy Tarreau5cf64dd2019-05-17 10:36:08 +0200192 chunk_appendf(buf,
Willy Tarreauf0e5da22020-05-01 12:26:03 +0200193 "%c%cThread %-2u: id=0x%llx act=%d glob=%d wq=%d rq=%d tl=%d tlsz=%d rqsz=%d\n"
Willy Tarreaua3870b72021-09-13 19:24:47 +0200194 " %2u/%-2u stuck=%d prof=%d",
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +0200195 (thr == tid && !from_signal) ? '*' : ' ', stuck ? '>' : ' ', thr + 1,
Willy Tarreauff64d3b2020-05-01 11:28:49 +0200196 ha_get_pthread_id(thr),
Olivier Houchardcfbb3e62019-05-29 19:22:43 +0200197 thread_has_tasks(),
Willy Tarreauc958c702022-06-16 15:59:36 +0200198 !eb_is_empty(&ha_thread_ctx[thr].rqueue_shared),
Willy Tarreau1a9c9222021-10-01 11:30:33 +0200199 !eb_is_empty(&ha_thread_ctx[thr].timers),
200 !eb_is_empty(&ha_thread_ctx[thr].rqueue),
201 !(LIST_ISEMPTY(&ha_thread_ctx[thr].tasklets[TL_URGENT]) &&
202 LIST_ISEMPTY(&ha_thread_ctx[thr].tasklets[TL_NORMAL]) &&
203 LIST_ISEMPTY(&ha_thread_ctx[thr].tasklets[TL_BULK]) &&
204 MT_LIST_ISEMPTY(&ha_thread_ctx[thr].shared_tasklet_list)),
205 ha_thread_ctx[thr].tasks_in_list,
206 ha_thread_ctx[thr].rq_total,
Willy Tarreau66ad98a2022-06-28 10:49:57 +0200207 ha_thread_info[thr].tgid, ha_thread_info[thr].ltid + 1,
Willy Tarreaue6a02fa2019-05-22 07:06:44 +0200208 stuck,
Willy Tarreaub30ced32023-05-04 11:30:55 +0200209 !!(ha_thread_ctx[thr].flags & TH_FL_TASK_PROFILING));
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200210
Willy Tarreau95e6c992023-05-07 15:02:30 +0200211#if defined(USE_THREAD)
Willy Tarreau5cf64dd2019-05-17 10:36:08 +0200212 chunk_appendf(buf,
Willy Tarreaub30ced32023-05-04 11:30:55 +0200213 " harmless=%d isolated=%d",
Willy Tarreau03f9b352022-06-27 16:02:24 +0200214 !!(_HA_ATOMIC_LOAD(&ha_tgroup_ctx[tgrp-1].threads_harmless) & thr_bit),
Willy Tarreaub30ced32023-05-04 11:30:55 +0200215 isolated_thread == thr);
Willy Tarreau95e6c992023-05-07 15:02:30 +0200216#endif
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200217
Willy Tarreau5cf64dd2019-05-17 10:36:08 +0200218 chunk_appendf(buf, "\n");
Willy Tarreau9c8800a2019-05-20 20:52:20 +0200219 chunk_appendf(buf, " cpu_ns: poll=%llu now=%llu diff=%llu\n", p, n, n-p);
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200220
Willy Tarreaua3870b72021-09-13 19:24:47 +0200221 /* this is the end of what we can dump from outside the current thread */
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200222
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +0200223 if (thr != tid && !thread_isolated())
224 goto leave;
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200225
Willy Tarreau5cf64dd2019-05-17 10:36:08 +0200226 chunk_appendf(buf, " curr_task=");
Willy Tarreau1a9c9222021-10-01 11:30:33 +0200227 ha_task_dump(buf, th_ctx->current, " ");
Willy Tarreauf5b4e062020-03-03 15:40:23 +0100228
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +0200229 if (stuck && thr == tid) {
Willy Tarreauf5b4e062020-03-03 15:40:23 +0100230 /* We only emit the backtrace for stuck threads in order not to
231 * waste precious output buffer space with non-interesting data.
Willy Tarreau123fc972021-01-22 13:52:41 +0100232 * Please leave this as the last instruction in this function
233 * so that the compiler uses tail merging and the current
234 * function does not appear in the stack.
Willy Tarreauf5b4e062020-03-03 15:40:23 +0100235 */
Willy Tarreau2bfce7e2021-01-22 14:48:34 +0100236 ha_dump_backtrace(buf, " ", 0);
Willy Tarreauf5b4e062020-03-03 15:40:23 +0100237 }
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +0200238 leave:
239 /* end of dump, setting the buffer to 0x1 will tell the caller we're done */
240 HA_ATOMIC_STORE(&ha_thread_ctx[thr].thread_dump_buffer, (void*)0x1UL);
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200241}
242
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +0200243/* Triggers a thread dump from thread <thr>, either directly if it's the
244 * current thread or if thread dump signals are not implemented, or by sending
245 * a signal if it's a remote one and the feature is supported. The buffer <buf>
246 * will get the dump appended, and the caller is responsible for making sure
247 * there is enough room otherwise some contents will be truncated.
248 */
249void ha_thread_dump(struct buffer *buf, int thr)
250{
251 struct buffer *old = NULL;
252
253 /* try to impose our dump buffer and to reserve the target thread's
254 * next dump for us.
255 */
256 do {
257 if (old)
258 ha_thread_relax();
259 old = NULL;
260 } while (!HA_ATOMIC_CAS(&ha_thread_ctx[thr].thread_dump_buffer, &old, buf));
261
262#ifdef USE_THREAD_DUMP
263 /* asking the remote thread to dump itself allows to get more details
264 * including a backtrace.
265 */
266 if (thr != tid)
267 ha_tkill(thr, DEBUGSIG);
268 else
269#endif
270 ha_thread_dump_one(thr, thr != tid);
271
272 /* now wait for the dump to be done, and release it */
273 do {
274 if (old)
275 ha_thread_relax();
276 old = (void*)0x01;
277 } while (!HA_ATOMIC_CAS(&ha_thread_ctx[thr].thread_dump_buffer, &old, 0));
278}
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200279
Willy Tarreau5cf64dd2019-05-17 10:36:08 +0200280/* dumps into the buffer some information related to task <task> (which may
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200281 * either be a task or a tasklet, and prepend each line except the first one
Willy Tarreau5cf64dd2019-05-17 10:36:08 +0200282 * with <pfx>. The buffer is only appended and the first output starts by the
283 * pointer itself. The caller is responsible for making sure the task is not
284 * going to vanish during the dump.
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200285 */
Willy Tarreau5cf64dd2019-05-17 10:36:08 +0200286void ha_task_dump(struct buffer *buf, const struct task *task, const char *pfx)
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200287{
Willy Tarreau578ea8b2019-05-22 09:43:09 +0200288 const struct stream *s = NULL;
Willy Tarreaua512b022019-08-21 14:12:19 +0200289 const struct appctx __maybe_unused *appctx = NULL;
Willy Tarreau78a7cb62019-08-21 14:16:02 +0200290 struct hlua __maybe_unused *hlua = NULL;
Willy Tarreau52f238d2022-07-15 12:08:40 +0200291 const struct stconn *sc;
Willy Tarreau578ea8b2019-05-22 09:43:09 +0200292
Willy Tarreau14a1ab72019-05-17 10:34:25 +0200293 if (!task) {
Willy Tarreau5cf64dd2019-05-17 10:36:08 +0200294 chunk_appendf(buf, "0\n");
Willy Tarreau231ec392019-05-17 10:39:47 +0200295 return;
296 }
297
Willy Tarreau20db9112019-05-17 14:14:35 +0200298 if (TASK_IS_TASKLET(task))
299 chunk_appendf(buf,
300 "%p (tasklet) calls=%u\n",
301 task,
302 task->calls);
303 else
304 chunk_appendf(buf,
305 "%p (task) calls=%u last=%llu%s\n",
306 task,
307 task->calls,
Willy Tarreau04e50b32022-09-07 14:49:50 +0200308 task->wake_date ? (unsigned long long)(now_mono_time() - task->wake_date) : 0,
309 task->wake_date ? " ns ago" : "");
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200310
Willy Tarreau2e89b092020-03-03 17:13:02 +0100311 chunk_appendf(buf, "%s fct=%p(", pfx, task->process);
312 resolve_sym_name(buf, NULL, task->process);
313 chunk_appendf(buf,") ctx=%p", task->context);
Willy Tarreau578ea8b2019-05-22 09:43:09 +0200314
Willy Tarreaua512b022019-08-21 14:12:19 +0200315 if (task->process == task_run_applet && (appctx = task->context))
316 chunk_appendf(buf, "(%s)\n", appctx->applet->name);
317 else
318 chunk_appendf(buf, "\n");
319
Willy Tarreau578ea8b2019-05-22 09:43:09 +0200320 if (task->process == process_stream && task->context)
321 s = (struct stream *)task->context;
Willy Tarreau52f238d2022-07-15 12:08:40 +0200322 else if (task->process == task_run_applet && task->context && (sc = appctx_sc((struct appctx *)task->context)))
323 s = sc_strm(sc);
Willy Tarreau462b9892022-05-18 18:06:53 +0200324 else if (task->process == sc_conn_io_cb && task->context)
Willy Tarreauea27f482022-05-18 16:10:52 +0200325 s = sc_strm(((struct stconn *)task->context));
Willy Tarreau578ea8b2019-05-22 09:43:09 +0200326
327 if (s)
328 stream_dump(buf, s, pfx, '\n');
Willy Tarreau78a7cb62019-08-21 14:16:02 +0200329
330#ifdef USE_LUA
331 hlua = NULL;
332 if (s && (hlua = s->hlua)) {
333 chunk_appendf(buf, "%sCurrent executing Lua from a stream analyser -- ", pfx);
334 }
335 else if (task->process == hlua_process_task && (hlua = task->context)) {
336 chunk_appendf(buf, "%sCurrent executing a Lua task -- ", pfx);
337 }
338 else if (task->process == task_run_applet && (appctx = task->context) &&
Willy Tarreaue23f33b2022-05-06 14:07:13 +0200339 (appctx->applet->fct == hlua_applet_tcp_fct)) {
Willy Tarreau78a7cb62019-08-21 14:16:02 +0200340 chunk_appendf(buf, "%sCurrent executing a Lua TCP service -- ", pfx);
341 }
342 else if (task->process == task_run_applet && (appctx = task->context) &&
Willy Tarreauaa229cc2022-05-06 14:26:10 +0200343 (appctx->applet->fct == hlua_applet_http_fct)) {
Willy Tarreau78a7cb62019-08-21 14:16:02 +0200344 chunk_appendf(buf, "%sCurrent executing a Lua HTTP service -- ", pfx);
345 }
346
Christopher Faulet471425f2020-07-24 19:08:05 +0200347 if (hlua && hlua->T) {
Christopher Fauletcc2c4f82021-03-24 14:52:24 +0100348 chunk_appendf(buf, "stack traceback:\n ");
349 append_prefixed_str(buf, hlua_traceback(hlua->T, "\n "), pfx, '\n', 0);
Willy Tarreau78a7cb62019-08-21 14:16:02 +0200350 }
Willy Tarreaucb01f5d2023-05-04 16:28:30 +0200351
352 /* we may need to terminate the current line */
353 if (*b_peek(buf, b_data(buf)-1) != '\n')
Christopher Faulet471425f2020-07-24 19:08:05 +0200354 b_putchr(buf, '\n');
Willy Tarreau78a7cb62019-08-21 14:16:02 +0200355#endif
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200356}
357
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200358/* This function dumps all profiling settings. It returns 0 if the output
359 * buffer is full and it needs to be called again, otherwise non-zero.
360 */
361static int cli_io_handler_show_threads(struct appctx *appctx)
362{
Willy Tarreauc12b3212022-05-27 11:08:15 +0200363 struct stconn *sc = appctx_sc(appctx);
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200364 int thr;
365
Christopher Faulet87633c32023-04-03 18:32:50 +0200366 /* FIXME: Don't watch the other side !*/
Christopher Faulet208c7122023-04-13 16:16:15 +0200367 if (unlikely(sc_opposite(sc)->flags & SC_FL_SHUT_DONE))
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200368 return 1;
369
370 if (appctx->st0)
371 thr = appctx->st1;
372 else
373 thr = 0;
374
Willy Tarreau009b5512023-05-04 19:07:56 +0200375 do {
376 chunk_reset(&trash);
377 ha_thread_dump(&trash, thr);
378
379 if (applet_putchk(appctx, &trash) == -1) {
380 /* failed, try again */
381 appctx->st1 = thr;
382 return 0;
383 }
384 thr++;
385 } while (thr < global.nbthread);
Willy Tarreau5cf64dd2019-05-17 10:36:08 +0200386
Willy Tarreau4e2b6462019-05-16 17:44:30 +0200387 return 1;
388}
389
Willy Tarreau6ab7b212021-12-28 09:57:10 +0100390#if defined(HA_HAVE_DUMP_LIBS)
391/* parse a "show libs" command. It returns 1 if it emits anything otherwise zero. */
392static int debug_parse_cli_show_libs(char **args, char *payload, struct appctx *appctx, void *private)
393{
394 if (!cli_has_level(appctx, ACCESS_LVL_OPER))
395 return 1;
396
397 chunk_reset(&trash);
398 if (dump_libs(&trash, 1))
399 return cli_msg(appctx, LOG_INFO, trash.area);
400 else
401 return 0;
402}
403#endif
404
Willy Tarreau880d1682023-05-04 18:52:51 +0200405/* Dumps a state of all threads into the trash and on fd #2, then aborts.
406 * A copy will be put into a trash chunk that's assigned to thread_dump_buffer
407 * so that the debugger can easily find it. This buffer might be truncated if
408 * too many threads are being dumped, but at least we'll dump them all on stderr.
409 * If thread_dump_buffer is set, it means that a panic has already begun.
410 */
Willy Tarreau56131ca2019-05-20 13:48:29 +0200411void ha_panic()
412{
Willy Tarreau880d1682023-05-04 18:52:51 +0200413 struct buffer *old;
414 unsigned int thr;
415
416 old = NULL;
417 if (!HA_ATOMIC_CAS(&thread_dump_buffer, &old, get_trash_chunk())) {
Willy Tarreau55433f92022-07-15 12:48:58 +0200418 /* a panic dump is already in progress, let's not disturb it,
419 * we'll be called via signal DEBUGSIG. By returning we may be
420 * able to leave a current signal handler (e.g. WDT) so that
421 * this will ensure more reliable signal delivery.
422 */
423 return;
424 }
Willy Tarreau880d1682023-05-04 18:52:51 +0200425
Willy Tarreau56131ca2019-05-20 13:48:29 +0200426 chunk_reset(&trash);
Willy Tarreaua9f9fc92019-05-20 17:45:35 +0200427 chunk_appendf(&trash, "Thread %u is about to kill the process.\n", tid + 1);
Willy Tarreau880d1682023-05-04 18:52:51 +0200428
429 for (thr = 0; thr < global.nbthread; thr++) {
430 ha_thread_dump(&trash, thr);
431 DISGUISE(write(2, trash.area, trash.data));
432 b_force_xfer(thread_dump_buffer, &trash, b_room(thread_dump_buffer));
433 chunk_reset(&trash);
434 }
435
Willy Tarreau56131ca2019-05-20 13:48:29 +0200436 for (;;)
437 abort();
438}
439
Willy Tarreau06e66c82022-03-02 15:52:03 +0100440/* Complain with message <msg> on stderr. If <counter> is not NULL, it is
441 * atomically incremented, and the message is only printed when the counter
442 * was zero, so that the message is only printed once. <taint> is only checked
443 * on bit 1, and will taint the process either for a bug (2) or warn (0).
444 */
445void complain(int *counter, const char *msg, int taint)
446{
447 if (counter && _HA_ATOMIC_FETCH_ADD(counter, 1))
448 return;
449 DISGUISE(write(2, msg, strlen(msg)));
450 if (taint & 2)
451 mark_tainted(TAINTED_BUG);
452 else
453 mark_tainted(TAINTED_WARN);
454}
455
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200456/* parse a "debug dev exit" command. It always returns 1, though it should never return. */
457static int debug_parse_cli_exit(char **args, char *payload, struct appctx *appctx, void *private)
458{
459 int code = atoi(args[3]);
460
461 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
462 return 1;
463
Willy Tarreau4781b152021-04-06 13:53:36 +0200464 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200465 exit(code);
466 return 1;
467}
468
Willy Tarreau5baf4fe2021-01-22 14:15:46 +0100469/* parse a "debug dev bug" command. It always returns 1, though it should never return.
470 * Note: we make sure not to make the function static so that it appears in the trace.
471 */
472int debug_parse_cli_bug(char **args, char *payload, struct appctx *appctx, void *private)
473{
474 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
475 return 1;
476
Willy Tarreau4781b152021-04-06 13:53:36 +0200477 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreau5baf4fe2021-01-22 14:15:46 +0100478 BUG_ON(one > zero);
479 return 1;
480}
481
Willy Tarreau305cfbd2022-02-25 08:52:39 +0100482/* parse a "debug dev warn" command. It always returns 1.
483 * Note: we make sure not to make the function static so that it appears in the trace.
484 */
485int debug_parse_cli_warn(char **args, char *payload, struct appctx *appctx, void *private)
486{
487 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
488 return 1;
489
490 _HA_ATOMIC_INC(&debug_commands_issued);
491 WARN_ON(one > zero);
492 return 1;
493}
494
Willy Tarreau6d3f1e32022-02-28 11:51:23 +0100495/* parse a "debug dev check" command. It always returns 1.
Willy Tarreau4e0a8b12022-02-25 08:55:11 +0100496 * Note: we make sure not to make the function static so that it appears in the trace.
497 */
Willy Tarreau6d3f1e32022-02-28 11:51:23 +0100498int debug_parse_cli_check(char **args, char *payload, struct appctx *appctx, void *private)
Willy Tarreau4e0a8b12022-02-25 08:55:11 +0100499{
500 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
501 return 1;
502
503 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreau6d3f1e32022-02-28 11:51:23 +0100504 CHECK_IF(one > zero);
Willy Tarreau4e0a8b12022-02-25 08:55:11 +0100505 return 1;
506}
507
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200508/* parse a "debug dev close" command. It always returns 1. */
509static int debug_parse_cli_close(char **args, char *payload, struct appctx *appctx, void *private)
510{
511 int fd;
512
513 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
514 return 1;
515
Willy Tarreau9d008692019-08-09 11:21:01 +0200516 if (!*args[3])
517 return cli_err(appctx, "Missing file descriptor number.\n");
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200518
519 fd = atoi(args[3]);
Willy Tarreau9d008692019-08-09 11:21:01 +0200520 if (fd < 0 || fd >= global.maxsock)
521 return cli_err(appctx, "File descriptor out of range.\n");
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200522
Willy Tarreau9d008692019-08-09 11:21:01 +0200523 if (!fdtab[fd].owner)
524 return cli_msg(appctx, LOG_INFO, "File descriptor was already closed.\n");
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200525
Willy Tarreau4781b152021-04-06 13:53:36 +0200526 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200527 fd_delete(fd);
528 return 1;
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200529}
530
Willy Tarreaudadf00e2022-07-15 08:25:03 +0200531/* this is meant to cause a deadlock when more than one task is running it or when run twice */
532static struct task *debug_run_cli_deadlock(struct task *task, void *ctx, unsigned int state)
533{
534 static HA_SPINLOCK_T lock __maybe_unused;
535
536 HA_SPIN_LOCK(OTHER_LOCK, &lock);
537 return NULL;
538}
539
540/* parse a "debug dev deadlock" command. It always returns 1. */
541static int debug_parse_cli_deadlock(char **args, char *payload, struct appctx *appctx, void *private)
542{
543 int tasks;
544
545 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
546 return 1;
547
548 _HA_ATOMIC_INC(&debug_commands_issued);
549 for (tasks = atoi(args[3]); tasks > 0; tasks--) {
550 struct task *t = task_new_on(tasks % global.nbthread);
551 if (!t)
552 continue;
553 t->process = debug_run_cli_deadlock;
554 t->context = NULL;
555 task_wakeup(t, TASK_WOKEN_INIT);
556 }
557
558 return 1;
559}
560
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200561/* parse a "debug dev delay" command. It always returns 1. */
562static int debug_parse_cli_delay(char **args, char *payload, struct appctx *appctx, void *private)
563{
564 int delay = atoi(args[3]);
565
566 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
567 return 1;
568
Willy Tarreau4781b152021-04-06 13:53:36 +0200569 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200570 usleep((long)delay * 1000);
571 return 1;
572}
573
574/* parse a "debug dev log" command. It always returns 1. */
575static int debug_parse_cli_log(char **args, char *payload, struct appctx *appctx, void *private)
576{
577 int arg;
578
579 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
580 return 1;
581
Willy Tarreau4781b152021-04-06 13:53:36 +0200582 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200583 chunk_reset(&trash);
584 for (arg = 3; *args[arg]; arg++) {
585 if (arg > 3)
586 chunk_strcat(&trash, " ");
587 chunk_strcat(&trash, args[arg]);
588 }
589
590 send_log(NULL, LOG_INFO, "%s\n", trash.area);
591 return 1;
592}
593
594/* parse a "debug dev loop" command. It always returns 1. */
595static int debug_parse_cli_loop(char **args, char *payload, struct appctx *appctx, void *private)
596{
597 struct timeval deadline, curr;
598 int loop = atoi(args[3]);
Willy Tarreaue5e62232023-05-04 11:50:26 +0200599 int isolate;
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200600
601 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
602 return 1;
603
Willy Tarreaue5e62232023-05-04 11:50:26 +0200604 isolate = strcmp(args[4], "isolated") == 0;
605
Willy Tarreau4781b152021-04-06 13:53:36 +0200606 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200607 gettimeofday(&curr, NULL);
608 tv_ms_add(&deadline, &curr, loop);
609
Willy Tarreaue5e62232023-05-04 11:50:26 +0200610 if (isolate)
611 thread_isolate();
612
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200613 while (tv_ms_cmp(&curr, &deadline) < 0)
614 gettimeofday(&curr, NULL);
615
Willy Tarreaue5e62232023-05-04 11:50:26 +0200616 if (isolate)
617 thread_release();
618
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200619 return 1;
620}
621
622/* parse a "debug dev panic" command. It always returns 1, though it should never return. */
623static int debug_parse_cli_panic(char **args, char *payload, struct appctx *appctx, void *private)
624{
625 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
626 return 1;
627
Willy Tarreau4781b152021-04-06 13:53:36 +0200628 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200629 ha_panic();
630 return 1;
631}
632
633/* parse a "debug dev exec" command. It always returns 1. */
Willy Tarreaub24ab222019-10-24 18:03:39 +0200634#if defined(DEBUG_DEV)
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200635static int debug_parse_cli_exec(char **args, char *payload, struct appctx *appctx, void *private)
636{
Willy Tarreau368bff42019-12-06 17:18:28 +0100637 int pipefd[2];
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200638 int arg;
Willy Tarreau368bff42019-12-06 17:18:28 +0100639 int pid;
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200640
641 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
642 return 1;
643
Willy Tarreau4781b152021-04-06 13:53:36 +0200644 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200645 chunk_reset(&trash);
646 for (arg = 3; *args[arg]; arg++) {
647 if (arg > 3)
648 chunk_strcat(&trash, " ");
649 chunk_strcat(&trash, args[arg]);
650 }
651
Willy Tarreau368bff42019-12-06 17:18:28 +0100652 thread_isolate();
653 if (pipe(pipefd) < 0)
654 goto fail_pipe;
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200655
Willy Tarreau38247432022-04-26 10:24:14 +0200656 if (fd_set_cloexec(pipefd[0]) == -1)
Willy Tarreau368bff42019-12-06 17:18:28 +0100657 goto fail_fcntl;
658
Willy Tarreau38247432022-04-26 10:24:14 +0200659 if (fd_set_cloexec(pipefd[1]) == -1)
Willy Tarreau368bff42019-12-06 17:18:28 +0100660 goto fail_fcntl;
661
662 pid = fork();
663
664 if (pid < 0)
665 goto fail_fork;
666 else if (pid == 0) {
667 /* child */
668 char *cmd[4] = { "/bin/sh", "-c", 0, 0 };
669
670 close(0);
671 dup2(pipefd[1], 1);
672 dup2(pipefd[1], 2);
673
674 cmd[2] = trash.area;
675 execvp(cmd[0], cmd);
676 printf("execvp() failed\n");
677 exit(1);
678 }
679
680 /* parent */
681 thread_release();
682 close(pipefd[1]);
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200683 chunk_reset(&trash);
684 while (1) {
Willy Tarreau368bff42019-12-06 17:18:28 +0100685 size_t ret = read(pipefd[0], trash.area + trash.data, trash.size - 20 - trash.data);
686 if (ret <= 0)
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200687 break;
688 trash.data += ret;
689 if (trash.data + 20 == trash.size) {
690 chunk_strcat(&trash, "\n[[[TRUNCATED]]]\n");
691 break;
692 }
693 }
Willy Tarreau368bff42019-12-06 17:18:28 +0100694 close(pipefd[0]);
695 waitpid(pid, NULL, WNOHANG);
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200696 trash.area[trash.data] = 0;
Willy Tarreau9d008692019-08-09 11:21:01 +0200697 return cli_msg(appctx, LOG_INFO, trash.area);
Willy Tarreau368bff42019-12-06 17:18:28 +0100698
699 fail_fork:
700 fail_fcntl:
701 close(pipefd[0]);
702 close(pipefd[1]);
703 fail_pipe:
704 thread_release();
705 return cli_err(appctx, "Failed to execute command.\n");
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200706}
Willy Tarreaubd3b44e2023-03-09 08:25:01 +0100707
708/* handles SIGRTMAX to inject random delays on the receiving thread in order
709 * to try to increase the likelihood to reproduce inter-thread races. The
710 * signal is periodically sent by a task initiated by "debug dev delay-inj".
711 */
712void debug_delay_inj_sighandler(int sig, siginfo_t *si, void *arg)
713{
714 volatile int i = statistical_prng_range(10000);
715
716 while (i--)
717 __ha_cpu_relax();
718}
Willy Tarreaub24ab222019-10-24 18:03:39 +0200719#endif
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200720
721/* parse a "debug dev hex" command. It always returns 1. */
722static int debug_parse_cli_hex(char **args, char *payload, struct appctx *appctx, void *private)
723{
724 unsigned long start, len;
725
726 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
727 return 1;
728
Willy Tarreau9d008692019-08-09 11:21:01 +0200729 if (!*args[3])
730 return cli_err(appctx, "Missing memory address to dump from.\n");
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200731
732 start = strtoul(args[3], NULL, 0);
Willy Tarreau9d008692019-08-09 11:21:01 +0200733 if (!start)
734 return cli_err(appctx, "Will not dump from NULL address.\n");
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200735
Willy Tarreau4781b152021-04-06 13:53:36 +0200736 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreau9b013702019-10-24 18:18:02 +0200737
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200738 /* by default, dump ~128 till next block of 16 */
739 len = strtoul(args[4], NULL, 0);
740 if (!len)
741 len = ((start + 128) & -16) - start;
742
743 chunk_reset(&trash);
Willy Tarreau37101052019-05-20 16:48:20 +0200744 dump_hex(&trash, " ", (const void *)start, len, 1);
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200745 trash.area[trash.data] = 0;
Willy Tarreau9d008692019-08-09 11:21:01 +0200746 return cli_msg(appctx, LOG_INFO, trash.area);
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200747}
748
Willy Tarreau48129be2021-05-04 18:40:50 +0200749/* parse a "debug dev sym <addr>" command. It always returns 1. */
750static int debug_parse_cli_sym(char **args, char *payload, struct appctx *appctx, void *private)
751{
752 unsigned long addr;
753
754 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
755 return 1;
756
757 if (!*args[3])
758 return cli_err(appctx, "Missing memory address to be resolved.\n");
759
760 _HA_ATOMIC_INC(&debug_commands_issued);
761
762 addr = strtoul(args[3], NULL, 0);
763 chunk_printf(&trash, "%#lx resolves to ", addr);
764 resolve_sym_name(&trash, NULL, (const void *)addr);
765 chunk_appendf(&trash, "\n");
766
767 return cli_msg(appctx, LOG_INFO, trash.area);
768}
769
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200770/* parse a "debug dev tkill" command. It always returns 1. */
771static int debug_parse_cli_tkill(char **args, char *payload, struct appctx *appctx, void *private)
772{
773 int thr = 0;
774 int sig = SIGABRT;
775
776 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
777 return 1;
778
779 if (*args[3])
780 thr = atoi(args[3]);
781
Willy Tarreau9d008692019-08-09 11:21:01 +0200782 if (thr < 0 || thr > global.nbthread)
783 return cli_err(appctx, "Thread number out of range (use 0 for current).\n");
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200784
785 if (*args[4])
786 sig = atoi(args[4]);
787
Willy Tarreau4781b152021-04-06 13:53:36 +0200788 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200789 if (thr)
Willy Tarreaufade80d2019-05-22 08:46:59 +0200790 ha_tkill(thr - 1, sig);
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200791 else
Willy Tarreau6bdf3e92019-05-20 14:25:05 +0200792 raise(sig);
793 return 1;
794}
795
Erwan Le Goas54966df2022-09-14 17:24:22 +0200796/* hashes 'word' in "debug dev hash 'word' ". */
797static int debug_parse_cli_hash(char **args, char *payload, struct appctx *appctx, void *private)
798{
799 char *msg = NULL;
800
801 cli_dynmsg(appctx, LOG_INFO, memprintf(&msg, "%s\n", HA_ANON_CLI(args[3])));
802 return 1;
803}
804
Willy Tarreau6cbe62b2020-03-05 17:16:24 +0100805/* parse a "debug dev write" command. It always returns 1. */
806static int debug_parse_cli_write(char **args, char *payload, struct appctx *appctx, void *private)
807{
808 unsigned long len;
809
810 if (!*args[3])
811 return cli_err(appctx, "Missing output size.\n");
812
813 len = strtoul(args[3], NULL, 0);
814 if (len >= trash.size)
815 return cli_err(appctx, "Output too large, must be <tune.bufsize.\n");
816
Willy Tarreau4781b152021-04-06 13:53:36 +0200817 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreau6cbe62b2020-03-05 17:16:24 +0100818
819 chunk_reset(&trash);
820 trash.data = len;
821 memset(trash.area, '.', trash.data);
822 trash.area[trash.data] = 0;
823 for (len = 64; len < trash.data; len += 64)
824 trash.area[len] = '\n';
825 return cli_msg(appctx, LOG_INFO, trash.area);
826}
827
Willy Tarreau68680bb2019-10-23 17:23:25 +0200828/* parse a "debug dev stream" command */
829/*
830 * debug dev stream [strm=<ptr>] [strm.f[{+-=}<flags>]] [txn.f[{+-=}<flags>]] \
831 * [req.f[{+-=}<flags>]] [res.f[{+-=}<flags>]] \
832 * [sif.f[{+-=<flags>]] [sib.f[{+-=<flags>]] \
833 * [sif.s[=<state>]] [sib.s[=<state>]]
834 */
835static int debug_parse_cli_stream(char **args, char *payload, struct appctx *appctx, void *private)
836{
Willy Tarreau0698c802022-05-11 14:09:57 +0200837 struct stream *s = appctx_strm(appctx);
Willy Tarreau68680bb2019-10-23 17:23:25 +0200838 int arg;
839 void *ptr;
840 int size;
841 const char *word, *end;
842 struct ist name;
843 char *msg = NULL;
844 char *endarg;
845 unsigned long long old, new;
846
847 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
848 return 1;
849
850 ptr = NULL; size = 0;
851
852 if (!*args[3]) {
853 return cli_err(appctx,
Willy Tarreaudd014482023-05-02 16:37:13 +0200854 "Usage: debug dev stream [ strm=<ptr> ] { <obj> <op> <value> | wake }*\n"
855 " <obj> = { strm.f | strm.x | scf.s | scb.s | txn.f | req.f | res.f }\n"
Willy Tarreau68680bb2019-10-23 17:23:25 +0200856 " <op> = {'' (show) | '=' (assign) | '^' (xor) | '+' (or) | '-' (andnot)}\n"
857 " <value> = 'now' | 64-bit dec/hex integer (0x prefix supported)\n"
858 " 'wake' wakes the stream asssigned to 'strm' (default: current)\n"
859 );
860 }
861
Willy Tarreau4781b152021-04-06 13:53:36 +0200862 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreau68680bb2019-10-23 17:23:25 +0200863 for (arg = 3; *args[arg]; arg++) {
864 old = 0;
865 end = word = args[arg];
866 while (*end && *end != '=' && *end != '^' && *end != '+' && *end != '-')
867 end++;
868 name = ist2(word, end - word);
869 if (isteq(name, ist("strm"))) {
Willy Tarreaub2fee042019-10-25 10:06:55 +0200870 ptr = (!s || !may_access(s)) ? NULL : &s; size = sizeof(s);
Willy Tarreau68680bb2019-10-23 17:23:25 +0200871 } else if (isteq(name, ist("strm.f"))) {
Willy Tarreaub2fee042019-10-25 10:06:55 +0200872 ptr = (!s || !may_access(s)) ? NULL : &s->flags; size = sizeof(s->flags);
Christopher Fauletae024ce2022-03-29 19:02:31 +0200873 } else if (isteq(name, ist("strm.x"))) {
874 ptr = (!s || !may_access(s)) ? NULL : &s->conn_exp; size = sizeof(s->conn_exp);
Willy Tarreau68680bb2019-10-23 17:23:25 +0200875 } else if (isteq(name, ist("txn.f"))) {
Willy Tarreaub2fee042019-10-25 10:06:55 +0200876 ptr = (!s || !may_access(s)) ? NULL : &s->txn->flags; size = sizeof(s->txn->flags);
Willy Tarreau68680bb2019-10-23 17:23:25 +0200877 } else if (isteq(name, ist("req.f"))) {
Willy Tarreaub2fee042019-10-25 10:06:55 +0200878 ptr = (!s || !may_access(s)) ? NULL : &s->req.flags; size = sizeof(s->req.flags);
Willy Tarreau68680bb2019-10-23 17:23:25 +0200879 } else if (isteq(name, ist("res.f"))) {
Willy Tarreaub2fee042019-10-25 10:06:55 +0200880 ptr = (!s || !may_access(s)) ? NULL : &s->res.flags; size = sizeof(s->res.flags);
Willy Tarreau7cb9e6c2022-05-17 19:40:40 +0200881 } else if (isteq(name, ist("scf.s"))) {
882 ptr = (!s || !may_access(s)) ? NULL : &s->scf->state; size = sizeof(s->scf->state);
883 } else if (isteq(name, ist("scb.s"))) {
884 ptr = (!s || !may_access(s)) ? NULL : &s->scf->state; size = sizeof(s->scb->state);
Willy Tarreau68680bb2019-10-23 17:23:25 +0200885 } else if (isteq(name, ist("wake"))) {
Willy Tarreau2b5520d2019-10-24 18:28:23 +0200886 if (s && may_access(s) && may_access((void *)s + sizeof(*s) - 1))
Willy Tarreau68680bb2019-10-23 17:23:25 +0200887 task_wakeup(s->task, TASK_WOKEN_TIMER|TASK_WOKEN_IO|TASK_WOKEN_MSG);
888 continue;
889 } else
890 return cli_dynerr(appctx, memprintf(&msg, "Unsupported field name: '%s'.\n", word));
891
892 /* read previous value */
Willy Tarreau2b5520d2019-10-24 18:28:23 +0200893 if ((s || ptr == &s) && ptr && may_access(ptr) && may_access(ptr + size - 1)) {
Willy Tarreau68680bb2019-10-23 17:23:25 +0200894 if (size == 8)
895 old = read_u64(ptr);
896 else if (size == 4)
897 old = read_u32(ptr);
898 else if (size == 2)
899 old = read_u16(ptr);
900 else
901 old = *(const uint8_t *)ptr;
Willy Tarreau2b5520d2019-10-24 18:28:23 +0200902 } else {
903 memprintf(&msg,
904 "%sSkipping inaccessible pointer %p for field '%.*s'.\n",
905 msg ? msg : "", ptr, (int)(end - word), word);
906 continue;
Willy Tarreau68680bb2019-10-23 17:23:25 +0200907 }
908
909 /* parse the new value . */
910 new = strtoll(end + 1, &endarg, 0);
911 if (end[1] && *endarg) {
912 if (strcmp(end + 1, "now") == 0)
913 new = now_ms;
914 else {
915 memprintf(&msg,
916 "%sIgnoring unparsable value '%s' for field '%.*s'.\n",
917 msg ? msg : "", end + 1, (int)(end - word), word);
918 continue;
919 }
920 }
921
922 switch (*end) {
923 case '\0': /* show */
924 memprintf(&msg, "%s%.*s=%#llx ", msg ? msg : "", (int)(end - word), word, old);
925 new = old; // do not change the value
926 break;
927
928 case '=': /* set */
929 break;
930
931 case '^': /* XOR */
932 new = old ^ new;
933 break;
934
935 case '+': /* OR */
936 new = old | new;
937 break;
938
939 case '-': /* AND NOT */
940 new = old & ~new;
941 break;
942
943 default:
944 break;
945 }
946
947 /* write the new value */
Willy Tarreau2b5520d2019-10-24 18:28:23 +0200948 if (new != old) {
Willy Tarreau68680bb2019-10-23 17:23:25 +0200949 if (size == 8)
950 write_u64(ptr, new);
951 else if (size == 4)
952 write_u32(ptr, new);
953 else if (size == 2)
954 write_u16(ptr, new);
955 else
956 *(uint8_t *)ptr = new;
957 }
958 }
959
960 if (msg && *msg)
961 return cli_dynmsg(appctx, LOG_INFO, msg);
962 return 1;
963}
964
Willy Tarreau98679872023-05-03 11:22:45 +0200965/* parse a "debug dev stream" command */
966/*
967 * debug dev task <ptr> [ "wake" | "expire" | "kill" ]
968 * Show/change status of a task/tasklet
969 */
970static int debug_parse_cli_task(char **args, char *payload, struct appctx *appctx, void *private)
971{
972 const struct ha_caller *caller;
973 struct task *t;
974 char *endarg;
975 char *msg;
976 void *ptr;
977 int ret = 1;
978 int task_ok;
979 int arg;
980
981 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
982 return 1;
983
984 /* parse the pointer value */
Willy Tarreau94df1b52023-05-12 04:40:06 +0200985 ptr = (void *)strtoul(args[3], &endarg, 0);
Willy Tarreau98679872023-05-03 11:22:45 +0200986 if (!*args[3] || *endarg)
987 goto usage;
988
989 _HA_ATOMIC_INC(&debug_commands_issued);
990
991 /* everything below must run under thread isolation till reaching label "leave" */
992 thread_isolate();
993
994 /* struct tasklet is smaller than struct task and is sufficient to check
995 * the TASK_COMMON part.
996 */
997 if (!may_access(ptr) || !may_access(ptr + sizeof(struct tasklet) - 1) ||
998 ((const struct tasklet *)ptr)->tid < -1 ||
999 ((const struct tasklet *)ptr)->tid >= (int)MAX_THREADS) {
1000 ret = cli_err(appctx, "The designated memory area doesn't look like a valid task/tasklet\n");
1001 goto leave;
1002 }
1003
1004 t = ptr;
1005 caller = t->caller;
1006 msg = NULL;
Aurelien DARRAGONb6a24a52023-05-15 11:59:08 +02001007 task_ok = may_access(ptr + sizeof(*t) - 1);
Willy Tarreau98679872023-05-03 11:22:45 +02001008
1009 chunk_reset(&trash);
1010 resolve_sym_name(&trash, NULL, (const void *)t->process);
1011
1012 /* we need to be careful here because we may dump a freed task that's
1013 * still in the pool cache, containing garbage in pointers.
1014 */
1015 if (!*args[4]) {
1016 memprintf(&msg, "%s%p: %s state=%#x tid=%d process=%s ctx=%p calls=%d last=%s:%d intl=%d",
1017 msg ? msg : "", t, (t->state & TASK_F_TASKLET) ? "tasklet" : "task",
1018 t->state, t->tid, trash.area, t->context, t->calls,
Willy Tarreauff508f12023-05-03 16:28:54 +02001019 caller && may_access(caller) && may_access(caller->func) && isalnum((uchar)*caller->func) ? caller->func : "0",
Willy Tarreau98679872023-05-03 11:22:45 +02001020 caller ? t->caller->line : 0,
1021 (t->state & TASK_F_TASKLET) ? LIST_INLIST(&((const struct tasklet *)t)->list) : 0);
1022
1023 if (task_ok && !(t->state & TASK_F_TASKLET))
1024 memprintf(&msg, "%s inrq=%d inwq=%d exp=%d nice=%d",
1025 msg ? msg : "", task_in_rq(t), task_in_wq(t), t->expire, t->nice);
1026
1027 memprintf(&msg, "%s\n", msg ? msg : "");
1028 }
1029
1030 for (arg = 4; *args[arg]; arg++) {
1031 if (strcmp(args[arg], "expire") == 0) {
1032 if (t->state & TASK_F_TASKLET) {
1033 /* do nothing for tasklets */
1034 }
1035 else if (task_ok) {
1036 /* unlink task and wake with timer flag */
1037 __task_unlink_wq(t);
1038 t->expire = now_ms;
1039 task_wakeup(t, TASK_WOKEN_TIMER);
1040 }
1041 } else if (strcmp(args[arg], "wake") == 0) {
1042 /* wake with all flags but init / timer */
1043 if (t->state & TASK_F_TASKLET)
1044 tasklet_wakeup((struct tasklet *)t);
1045 else if (task_ok)
1046 task_wakeup(t, TASK_WOKEN_ANY & ~(TASK_WOKEN_INIT|TASK_WOKEN_TIMER));
1047 } else if (strcmp(args[arg], "kill") == 0) {
1048 /* Kill the task. This is not idempotent! */
1049 if (!(t->state & TASK_KILLED)) {
1050 if (t->state & TASK_F_TASKLET)
1051 tasklet_kill((struct tasklet *)t);
1052 else if (task_ok)
1053 task_kill(t);
1054 }
1055 } else {
1056 thread_release();
1057 goto usage;
1058 }
1059 }
1060
1061 if (msg && *msg)
1062 ret = cli_dynmsg(appctx, LOG_INFO, msg);
1063 leave:
1064 thread_release();
1065 return ret;
1066 usage:
1067 return cli_err(appctx,
1068 "Usage: debug dev task <ptr> [ wake | expire | kill ]\n"
1069 " By default, dumps some info on task/tasklet <ptr>. 'wake' will wake it up\n"
1070 " with all conditions flags but init/exp. 'expire' will expire the entry, and\n"
1071 " 'kill' will kill it (warning: may crash since later not idempotent!). All\n"
1072 " changes may crash the process if performed on a wrong object!\n"
1073 );
1074}
1075
Willy Tarreaubd3b44e2023-03-09 08:25:01 +01001076#if defined(DEBUG_DEV)
1077static struct task *debug_delay_inj_task(struct task *t, void *ctx, unsigned int state)
1078{
1079 unsigned long *tctx = ctx; // [0] = interval, [1] = nbwakeups
1080 unsigned long inter = tctx[0];
1081 unsigned long count = tctx[1];
1082 unsigned long rnd;
1083
1084 if (inter)
1085 t->expire = tick_add(now_ms, inter);
1086 else
1087 task_wakeup(t, TASK_WOKEN_MSG);
1088
1089 /* wake a random thread */
1090 while (count--) {
1091 rnd = statistical_prng_range(global.nbthread);
1092 ha_tkill(rnd, SIGRTMAX);
1093 }
1094 return t;
1095}
1096
1097/* parse a "debug dev delay-inj" command
1098 * debug dev delay-inj <inter> <count>
1099 */
1100static int debug_parse_delay_inj(char **args, char *payload, struct appctx *appctx, void *private)
1101{
1102 unsigned long *tctx; // [0] = inter, [2] = count
1103 struct task *task;
1104
1105 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
1106 return 1;
1107
1108 if (!*args[4])
1109 return cli_err(appctx, "Usage: debug dev delay-inj <inter_ms> <count>*\n");
1110
1111 _HA_ATOMIC_INC(&debug_commands_issued);
1112
1113 tctx = calloc(sizeof(*tctx), 2);
1114 if (!tctx)
1115 goto fail;
1116
1117 tctx[0] = atoi(args[3]);
1118 tctx[1] = atoi(args[4]);
1119
1120 task = task_new_here/*anywhere*/();
1121 if (!task)
1122 goto fail;
1123
1124 task->process = debug_delay_inj_task;
1125 task->context = tctx;
1126 task_wakeup(task, TASK_WOKEN_INIT);
1127 return 1;
1128
1129 fail:
1130 free(tctx);
1131 return cli_err(appctx, "Not enough memory");
1132}
1133#endif // DEBUG_DEV
1134
Willy Tarreau144f84a2021-03-02 16:09:26 +01001135static struct task *debug_task_handler(struct task *t, void *ctx, unsigned int state)
Willy Tarreaua5a44792020-11-29 17:12:15 +01001136{
1137 unsigned long *tctx = ctx; // [0] = #tasks, [1] = inter, [2+] = { tl | (tsk+1) }
1138 unsigned long inter = tctx[1];
1139 unsigned long rnd;
1140
1141 t->expire = tick_add(now_ms, inter);
1142
1143 /* half of the calls will wake up another entry */
Willy Tarreau06e69b52021-03-02 14:01:35 +01001144 rnd = statistical_prng();
Willy Tarreaua5a44792020-11-29 17:12:15 +01001145 if (rnd & 1) {
1146 rnd >>= 1;
1147 rnd %= tctx[0];
1148 rnd = tctx[rnd + 2];
1149
1150 if (rnd & 1)
1151 task_wakeup((struct task *)(rnd - 1), TASK_WOKEN_MSG);
1152 else
1153 tasklet_wakeup((struct tasklet *)rnd);
1154 }
1155 return t;
1156}
1157
Willy Tarreau144f84a2021-03-02 16:09:26 +01001158static struct task *debug_tasklet_handler(struct task *t, void *ctx, unsigned int state)
Willy Tarreaua5a44792020-11-29 17:12:15 +01001159{
1160 unsigned long *tctx = ctx; // [0] = #tasks, [1] = inter, [2+] = { tl | (tsk+1) }
1161 unsigned long rnd;
1162 int i;
1163
1164 /* wake up two random entries */
1165 for (i = 0; i < 2; i++) {
Willy Tarreau06e69b52021-03-02 14:01:35 +01001166 rnd = statistical_prng() % tctx[0];
Willy Tarreaua5a44792020-11-29 17:12:15 +01001167 rnd = tctx[rnd + 2];
1168
1169 if (rnd & 1)
1170 task_wakeup((struct task *)(rnd - 1), TASK_WOKEN_MSG);
1171 else
1172 tasklet_wakeup((struct tasklet *)rnd);
1173 }
1174 return t;
1175}
1176
1177/* parse a "debug dev sched" command
1178 * debug dev sched {task|tasklet} [count=<count>] [mask=<mask>] [single=<single>] [inter=<inter>]
1179 */
1180static int debug_parse_cli_sched(char **args, char *payload, struct appctx *appctx, void *private)
1181{
1182 int arg;
1183 void *ptr;
1184 int size;
1185 const char *word, *end;
1186 struct ist name;
1187 char *msg = NULL;
1188 char *endarg;
1189 unsigned long long new;
1190 unsigned long count = 0;
Willy Tarreau8e5c53a2022-06-15 16:32:41 +02001191 unsigned long thrid = tid;
Willy Tarreaua5a44792020-11-29 17:12:15 +01001192 unsigned int inter = 0;
Willy Tarreaua5a44792020-11-29 17:12:15 +01001193 unsigned long i;
1194 int mode = 0; // 0 = tasklet; 1 = task
Willy Tarreaua5a44792020-11-29 17:12:15 +01001195 unsigned long *tctx; // [0] = #tasks, [1] = inter, [2+] = { tl | (tsk+1) }
1196
1197 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
1198 return 1;
1199
1200 ptr = NULL; size = 0;
Willy Tarreaua5a44792020-11-29 17:12:15 +01001201
1202 if (strcmp(args[3], "task") != 0 && strcmp(args[3], "tasklet") != 0) {
1203 return cli_err(appctx,
1204 "Usage: debug dev sched {task|tasklet} { <obj> = <value> }*\n"
Willy Tarreau8e5c53a2022-06-15 16:32:41 +02001205 " <obj> = {count | tid | inter }\n"
Willy Tarreaua5a44792020-11-29 17:12:15 +01001206 " <value> = 64-bit dec/hex integer (0x prefix supported)\n"
1207 );
1208 }
1209
1210 mode = strcmp(args[3], "task") == 0;
1211
Willy Tarreau4781b152021-04-06 13:53:36 +02001212 _HA_ATOMIC_INC(&debug_commands_issued);
Willy Tarreaua5a44792020-11-29 17:12:15 +01001213 for (arg = 4; *args[arg]; arg++) {
1214 end = word = args[arg];
1215 while (*end && *end != '=' && *end != '^' && *end != '+' && *end != '-')
1216 end++;
1217 name = ist2(word, end - word);
1218 if (isteq(name, ist("count"))) {
1219 ptr = &count; size = sizeof(count);
Willy Tarreaua5a44792020-11-29 17:12:15 +01001220 } else if (isteq(name, ist("tid"))) {
1221 ptr = &thrid; size = sizeof(thrid);
1222 } else if (isteq(name, ist("inter"))) {
1223 ptr = &inter; size = sizeof(inter);
Willy Tarreaua5a44792020-11-29 17:12:15 +01001224 } else
1225 return cli_dynerr(appctx, memprintf(&msg, "Unsupported setting: '%s'.\n", word));
1226
1227 /* parse the new value . */
1228 new = strtoll(end + 1, &endarg, 0);
1229 if (end[1] && *endarg) {
1230 memprintf(&msg,
1231 "%sIgnoring unparsable value '%s' for field '%.*s'.\n",
1232 msg ? msg : "", end + 1, (int)(end - word), word);
1233 continue;
1234 }
1235
1236 /* write the new value */
1237 if (size == 8)
1238 write_u64(ptr, new);
1239 else if (size == 4)
1240 write_u32(ptr, new);
1241 else if (size == 2)
1242 write_u16(ptr, new);
1243 else
1244 *(uint8_t *)ptr = new;
1245 }
1246
1247 tctx = calloc(sizeof(*tctx), count + 2);
1248 if (!tctx)
1249 goto fail;
1250
1251 tctx[0] = (unsigned long)count;
1252 tctx[1] = (unsigned long)inter;
1253
Willy Tarreau8e5c53a2022-06-15 16:32:41 +02001254 if (thrid >= global.nbthread)
1255 thrid = tid;
Willy Tarreaua5a44792020-11-29 17:12:15 +01001256
Willy Tarreaua5a44792020-11-29 17:12:15 +01001257 for (i = 0; i < count; i++) {
Willy Tarreaua5a44792020-11-29 17:12:15 +01001258 /* now, if poly or mask was set, tmask corresponds to the
1259 * valid thread mask to use, otherwise it remains zero.
1260 */
1261 //printf("%lu: mode=%d mask=%#lx\n", i, mode, tmask);
1262 if (mode == 0) {
1263 struct tasklet *tl = tasklet_new();
1264
1265 if (!tl)
1266 goto fail;
1267
Willy Tarreau8e5c53a2022-06-15 16:32:41 +02001268 tl->tid = thrid;
Willy Tarreaua5a44792020-11-29 17:12:15 +01001269 tl->process = debug_tasklet_handler;
1270 tl->context = tctx;
1271 tctx[i + 2] = (unsigned long)tl;
1272 } else {
Willy Tarreau8e5c53a2022-06-15 16:32:41 +02001273 struct task *task = task_new_on(thrid);
Willy Tarreaua5a44792020-11-29 17:12:15 +01001274
1275 if (!task)
1276 goto fail;
1277
1278 task->process = debug_task_handler;
1279 task->context = tctx;
1280 tctx[i + 2] = (unsigned long)task + 1;
1281 }
1282 }
1283
1284 /* start the tasks and tasklets */
1285 for (i = 0; i < count; i++) {
1286 unsigned long ctx = tctx[i + 2];
1287
1288 if (ctx & 1)
1289 task_wakeup((struct task *)(ctx - 1), TASK_WOKEN_INIT);
1290 else
1291 tasklet_wakeup((struct tasklet *)ctx);
1292 }
1293
1294 if (msg && *msg)
1295 return cli_dynmsg(appctx, LOG_INFO, msg);
1296 return 1;
1297
1298 fail:
1299 /* free partially allocated entries */
1300 for (i = 0; tctx && i < count; i++) {
1301 unsigned long ctx = tctx[i + 2];
1302
1303 if (!ctx)
1304 break;
1305
1306 if (ctx & 1)
1307 task_destroy((struct task *)(ctx - 1));
1308 else
1309 tasklet_free((struct tasklet *)ctx);
1310 }
1311
1312 free(tctx);
1313 return cli_err(appctx, "Not enough memory");
1314}
1315
Willy Tarreaue06bbf32022-05-05 14:26:28 +02001316/* CLI state for "debug dev fd" */
1317struct dev_fd_ctx {
1318 int start_fd;
1319};
1320
Willy Tarreau5be7c192022-01-24 20:26:09 +01001321/* CLI parser for the "debug dev fd" command. The current FD to restart from is
Willy Tarreaue06bbf32022-05-05 14:26:28 +02001322 * stored in a struct dev_fd_ctx pointed to by svcctx.
Willy Tarreau5be7c192022-01-24 20:26:09 +01001323 */
1324static int debug_parse_cli_fd(char **args, char *payload, struct appctx *appctx, void *private)
1325{
Willy Tarreaue06bbf32022-05-05 14:26:28 +02001326 struct dev_fd_ctx *ctx = applet_reserve_svcctx(appctx, sizeof(*ctx));
1327
Willy Tarreau5be7c192022-01-24 20:26:09 +01001328 if (!cli_has_level(appctx, ACCESS_LVL_OPER))
1329 return 1;
1330
1331 /* start at fd #0 */
Willy Tarreaue06bbf32022-05-05 14:26:28 +02001332 ctx->start_fd = 0;
Willy Tarreau5be7c192022-01-24 20:26:09 +01001333 return 0;
1334}
1335
1336/* CLI I/O handler for the "debug dev fd" command. Dumps all FDs that are
1337 * accessible from the process but not known from fdtab. The FD number to
Willy Tarreaue06bbf32022-05-05 14:26:28 +02001338 * restart from is stored in a struct dev_fd_ctx pointed to by svcctx.
Willy Tarreau5be7c192022-01-24 20:26:09 +01001339 */
1340static int debug_iohandler_fd(struct appctx *appctx)
1341{
Willy Tarreaue06bbf32022-05-05 14:26:28 +02001342 struct dev_fd_ctx *ctx = appctx->svcctx;
Willy Tarreauc12b3212022-05-27 11:08:15 +02001343 struct stconn *sc = appctx_sc(appctx);
Willy Tarreau5be7c192022-01-24 20:26:09 +01001344 struct sockaddr_storage sa;
1345 struct stat statbuf;
1346 socklen_t salen, vlen;
1347 int ret1, ret2, port;
1348 char *addrstr;
1349 int ret = 1;
1350 int i, fd;
1351
Christopher Faulet87633c32023-04-03 18:32:50 +02001352 /* FIXME: Don't watch the other side !*/
Christopher Faulet208c7122023-04-13 16:16:15 +02001353 if (unlikely(sc_opposite(sc)->flags & SC_FL_SHUT_DONE))
Willy Tarreau5be7c192022-01-24 20:26:09 +01001354 goto end;
1355
1356 chunk_reset(&trash);
1357
1358 thread_isolate();
1359
1360 /* we have two inner loops here, one for the proxy, the other one for
1361 * the buffer.
1362 */
Willy Tarreaue06bbf32022-05-05 14:26:28 +02001363 for (fd = ctx->start_fd; fd < global.maxsock; fd++) {
Willy Tarreau5be7c192022-01-24 20:26:09 +01001364 /* check for FD's existence */
1365 ret1 = fcntl(fd, F_GETFD, 0);
1366 if (ret1 == -1)
1367 continue; // not known to the process
1368 if (fdtab[fd].owner)
1369 continue; // well-known
1370
1371 /* OK we're seeing an orphan let's try to retrieve as much
1372 * information as possible about it.
1373 */
1374 chunk_printf(&trash, "%5d", fd);
1375
1376 if (fstat(fd, &statbuf) != -1) {
1377 chunk_appendf(&trash, " type=%s mod=%04o dev=%#llx siz=%#llx uid=%lld gid=%lld fs=%#llx ino=%#llx",
1378 isatty(fd) ? "tty.":
1379 S_ISREG(statbuf.st_mode) ? "file":
1380 S_ISDIR(statbuf.st_mode) ? "dir.":
1381 S_ISCHR(statbuf.st_mode) ? "chr.":
1382 S_ISBLK(statbuf.st_mode) ? "blk.":
1383 S_ISFIFO(statbuf.st_mode) ? "pipe":
1384 S_ISLNK(statbuf.st_mode) ? "link":
1385 S_ISSOCK(statbuf.st_mode) ? "sock":
1386#ifdef USE_EPOLL
1387 epoll_wait(fd, NULL, 0, 0) != -1 || errno != EBADF ? "epol":
1388#endif
1389 "????",
1390 (uint)statbuf.st_mode & 07777,
1391
1392 (ullong)statbuf.st_rdev,
1393 (ullong)statbuf.st_size,
1394 (ullong)statbuf.st_uid,
1395 (ullong)statbuf.st_gid,
1396
1397 (ullong)statbuf.st_dev,
1398 (ullong)statbuf.st_ino);
1399 }
1400
1401 chunk_appendf(&trash, " getfd=%s+%#x",
1402 (ret1 & FD_CLOEXEC) ? "cloex" : "",
1403 ret1 &~ FD_CLOEXEC);
1404
1405 /* FD options */
1406 ret2 = fcntl(fd, F_GETFL, 0);
1407 if (ret2) {
1408 chunk_appendf(&trash, " getfl=%s",
1409 (ret1 & 3) >= 2 ? "O_RDWR" :
1410 (ret1 & 1) ? "O_WRONLY" : "O_RDONLY");
1411
1412 for (i = 2; i < 32; i++) {
1413 if (!(ret2 & (1UL << i)))
1414 continue;
1415 switch (1UL << i) {
1416 case O_CREAT: chunk_appendf(&trash, ",O_CREAT"); break;
1417 case O_EXCL: chunk_appendf(&trash, ",O_EXCL"); break;
1418 case O_NOCTTY: chunk_appendf(&trash, ",O_NOCTTY"); break;
1419 case O_TRUNC: chunk_appendf(&trash, ",O_TRUNC"); break;
1420 case O_APPEND: chunk_appendf(&trash, ",O_APPEND"); break;
Willy Tarreau410942b2022-01-25 14:51:53 +01001421#ifdef O_ASYNC
Willy Tarreau5be7c192022-01-24 20:26:09 +01001422 case O_ASYNC: chunk_appendf(&trash, ",O_ASYNC"); break;
Willy Tarreau410942b2022-01-25 14:51:53 +01001423#endif
Willy Tarreau5be7c192022-01-24 20:26:09 +01001424#ifdef O_DIRECT
1425 case O_DIRECT: chunk_appendf(&trash, ",O_DIRECT"); break;
1426#endif
1427#ifdef O_NOATIME
1428 case O_NOATIME: chunk_appendf(&trash, ",O_NOATIME"); break;
1429#endif
1430 }
1431 }
1432 }
1433
1434 vlen = sizeof(ret2);
1435 ret1 = getsockopt(fd, SOL_SOCKET, SO_TYPE, &ret2, &vlen);
1436 if (ret1 != -1)
1437 chunk_appendf(&trash, " so_type=%d", ret2);
1438
1439 vlen = sizeof(ret2);
1440 ret1 = getsockopt(fd, SOL_SOCKET, SO_ACCEPTCONN, &ret2, &vlen);
1441 if (ret1 != -1)
1442 chunk_appendf(&trash, " so_accept=%d", ret2);
1443
1444 vlen = sizeof(ret2);
1445 ret1 = getsockopt(fd, SOL_SOCKET, SO_ERROR, &ret2, &vlen);
1446 if (ret1 != -1)
1447 chunk_appendf(&trash, " so_error=%d", ret2);
1448
1449 salen = sizeof(sa);
1450 if (getsockname(fd, (struct sockaddr *)&sa, &salen) != -1) {
1451 if (sa.ss_family == AF_INET)
1452 port = ntohs(((const struct sockaddr_in *)&sa)->sin_port);
1453 else if (sa.ss_family == AF_INET6)
1454 port = ntohs(((const struct sockaddr_in6 *)&sa)->sin6_port);
1455 else
1456 port = 0;
1457 addrstr = sa2str(&sa, port, 0);
1458 chunk_appendf(&trash, " laddr=%s", addrstr);
1459 free(addrstr);
1460 }
1461
1462 salen = sizeof(sa);
1463 if (getpeername(fd, (struct sockaddr *)&sa, &salen) != -1) {
1464 if (sa.ss_family == AF_INET)
1465 port = ntohs(((const struct sockaddr_in *)&sa)->sin_port);
1466 else if (sa.ss_family == AF_INET6)
1467 port = ntohs(((const struct sockaddr_in6 *)&sa)->sin6_port);
1468 else
1469 port = 0;
1470 addrstr = sa2str(&sa, port, 0);
1471 chunk_appendf(&trash, " raddr=%s", addrstr);
1472 free(addrstr);
1473 }
1474
1475 chunk_appendf(&trash, "\n");
1476
Willy Tarreaud0a06d52022-05-18 15:07:19 +02001477 if (applet_putchk(appctx, &trash) == -1) {
Willy Tarreaue06bbf32022-05-05 14:26:28 +02001478 ctx->start_fd = fd;
Willy Tarreau5be7c192022-01-24 20:26:09 +01001479 ret = 0;
1480 break;
1481 }
1482 }
1483
1484 thread_release();
1485 end:
1486 return ret;
1487}
1488
Willy Tarreaua6026a02020-07-02 09:14:48 +02001489#if defined(DEBUG_MEM_STATS)
Willy Tarreau40e952f2022-05-05 14:39:51 +02001490
1491/* CLI state for "debug dev memstats" */
1492struct dev_mem_ctx {
1493 struct mem_stats *start, *stop; /* begin/end of dump */
Willy Tarreaue57fbed2022-11-30 16:42:54 +01001494 char *match; /* non-null if a name prefix is specified */
Willy Tarreau40e952f2022-05-05 14:39:51 +02001495 int show_all; /* show all entries if non-null */
Willy Tarreaub59e3f62022-11-30 17:16:51 +01001496 int width; /* 1st column width */
1497 long tot_size; /* sum of alloc-free */
1498 ulong tot_calls; /* sum of calls */
Willy Tarreau40e952f2022-05-05 14:39:51 +02001499};
1500
1501/* CLI parser for the "debug dev memstats" command. Sets a dev_mem_ctx shown above. */
Willy Tarreaua6026a02020-07-02 09:14:48 +02001502static int debug_parse_cli_memstats(char **args, char *payload, struct appctx *appctx, void *private)
1503{
Willy Tarreau40e952f2022-05-05 14:39:51 +02001504 struct dev_mem_ctx *ctx = applet_reserve_svcctx(appctx, sizeof(*ctx));
Willy Tarreau334d0912022-11-30 16:50:48 +01001505 int arg;
Willy Tarreau40e952f2022-05-05 14:39:51 +02001506
Willy Tarreaua6026a02020-07-02 09:14:48 +02001507 extern __attribute__((__weak__)) struct mem_stats __start_mem_stats;
1508 extern __attribute__((__weak__)) struct mem_stats __stop_mem_stats;
1509
1510 if (!cli_has_level(appctx, ACCESS_LVL_OPER))
1511 return 1;
1512
Willy Tarreau334d0912022-11-30 16:50:48 +01001513 for (arg = 3; *args[arg]; arg++) {
1514 if (strcmp(args[arg], "reset") == 0) {
1515 struct mem_stats *ptr;
Willy Tarreaua6026a02020-07-02 09:14:48 +02001516
Willy Tarreau334d0912022-11-30 16:50:48 +01001517 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
1518 return 1;
Willy Tarreaua6026a02020-07-02 09:14:48 +02001519
Willy Tarreau334d0912022-11-30 16:50:48 +01001520 for (ptr = &__start_mem_stats; ptr < &__stop_mem_stats; ptr++) {
1521 _HA_ATOMIC_STORE(&ptr->calls, 0);
1522 _HA_ATOMIC_STORE(&ptr->size, 0);
1523 }
1524 return 1;
Willy Tarreaua6026a02020-07-02 09:14:48 +02001525 }
Willy Tarreau334d0912022-11-30 16:50:48 +01001526 else if (strcmp(args[arg], "all") == 0) {
1527 ctx->show_all = 1;
1528 continue;
1529 }
Willy Tarreaue57fbed2022-11-30 16:42:54 +01001530 else if (strcmp(args[arg], "match") == 0 && *args[arg + 1]) {
1531 ha_free(&ctx->match);
1532 ctx->match = strdup(args[arg + 1]);
1533 arg++;
1534 continue;
1535 }
Willy Tarreau334d0912022-11-30 16:50:48 +01001536 else
Willy Tarreaue57fbed2022-11-30 16:42:54 +01001537 return cli_err(appctx, "Expects either 'reset', 'all', or 'match <pfx>'.\n");
Willy Tarreaua6026a02020-07-02 09:14:48 +02001538 }
1539
Willy Tarreaua6026a02020-07-02 09:14:48 +02001540 /* otherwise proceed with the dump from p0 to p1 */
Willy Tarreau40e952f2022-05-05 14:39:51 +02001541 ctx->start = &__start_mem_stats;
1542 ctx->stop = &__stop_mem_stats;
Willy Tarreau4a426e22022-08-09 08:51:08 +02001543 ctx->width = 0;
Willy Tarreaua6026a02020-07-02 09:14:48 +02001544 return 0;
1545}
1546
Willy Tarreau40e952f2022-05-05 14:39:51 +02001547/* CLI I/O handler for the "debug dev memstats" command using a dev_mem_ctx
1548 * found in appctx->svcctx. Dumps all mem_stats structs referenced by pointers
1549 * located between ->start and ->stop. Dumps all entries if ->show_all != 0,
1550 * otherwise only non-zero calls.
Willy Tarreaua6026a02020-07-02 09:14:48 +02001551 */
1552static int debug_iohandler_memstats(struct appctx *appctx)
1553{
Willy Tarreau40e952f2022-05-05 14:39:51 +02001554 struct dev_mem_ctx *ctx = appctx->svcctx;
Willy Tarreauc12b3212022-05-27 11:08:15 +02001555 struct stconn *sc = appctx_sc(appctx);
Willy Tarreau4a426e22022-08-09 08:51:08 +02001556 struct mem_stats *ptr;
Willy Tarreaue57fbed2022-11-30 16:42:54 +01001557 const char *pfx = ctx->match;
Willy Tarreaua6026a02020-07-02 09:14:48 +02001558 int ret = 1;
1559
Christopher Faulet87633c32023-04-03 18:32:50 +02001560 /* FIXME: Don't watch the other side !*/
Christopher Faulet208c7122023-04-13 16:16:15 +02001561 if (unlikely(sc_opposite(sc)->flags & SC_FL_SHUT_DONE))
Willy Tarreaua6026a02020-07-02 09:14:48 +02001562 goto end;
1563
Willy Tarreau4a426e22022-08-09 08:51:08 +02001564 if (!ctx->width) {
1565 /* we don't know the first column's width, let's compute it
1566 * now based on a first pass on printable entries and their
1567 * expected width (approximated).
1568 */
1569 for (ptr = ctx->start; ptr != ctx->stop; ptr++) {
1570 const char *p, *name;
1571 int w = 0;
1572 char tmp;
1573
1574 if (!ptr->size && !ptr->calls && !ctx->show_all)
1575 continue;
1576
Willy Tarreaud96d2142022-09-06 08:05:59 +02001577 for (p = name = ptr->caller.file; *p; p++) {
Willy Tarreau4a426e22022-08-09 08:51:08 +02001578 if (*p == '/')
1579 name = p + 1;
1580 }
1581
1582 if (ctx->show_all)
Willy Tarreaud96d2142022-09-06 08:05:59 +02001583 w = snprintf(&tmp, 0, "%s(%s:%d) ", ptr->caller.func, name, ptr->caller.line);
Willy Tarreau4a426e22022-08-09 08:51:08 +02001584 else
Willy Tarreaud96d2142022-09-06 08:05:59 +02001585 w = snprintf(&tmp, 0, "%s:%d ", name, ptr->caller.line);
Willy Tarreau4a426e22022-08-09 08:51:08 +02001586
1587 if (w > ctx->width)
1588 ctx->width = w;
1589 }
1590 }
1591
Willy Tarreaua6026a02020-07-02 09:14:48 +02001592 /* we have two inner loops here, one for the proxy, the other one for
1593 * the buffer.
1594 */
Willy Tarreau4a426e22022-08-09 08:51:08 +02001595 for (ptr = ctx->start; ptr != ctx->stop; ptr++) {
Willy Tarreaua6026a02020-07-02 09:14:48 +02001596 const char *type;
1597 const char *name;
1598 const char *p;
Willy Tarreau55c950b2022-08-09 08:15:27 +02001599 const char *info = NULL;
Willy Tarreau17200dd2022-08-09 08:40:08 +02001600 const char *func = NULL;
Willy Tarreaub59e3f62022-11-30 17:16:51 +01001601 int direction = 0; // neither alloc nor free (e.g. realloc)
Willy Tarreaua6026a02020-07-02 09:14:48 +02001602
Willy Tarreau40e952f2022-05-05 14:39:51 +02001603 if (!ptr->size && !ptr->calls && !ctx->show_all)
Willy Tarreaua6026a02020-07-02 09:14:48 +02001604 continue;
1605
1606 /* basename only */
Willy Tarreaud96d2142022-09-06 08:05:59 +02001607 for (p = name = ptr->caller.file; *p; p++) {
Willy Tarreaua6026a02020-07-02 09:14:48 +02001608 if (*p == '/')
1609 name = p + 1;
1610 }
1611
Willy Tarreaud96d2142022-09-06 08:05:59 +02001612 func = ptr->caller.func;
Willy Tarreau17200dd2022-08-09 08:40:08 +02001613
Willy Tarreaud96d2142022-09-06 08:05:59 +02001614 switch (ptr->caller.what) {
Willy Tarreaub59e3f62022-11-30 17:16:51 +01001615 case MEM_STATS_TYPE_CALLOC: type = "CALLOC"; direction = 1; break;
1616 case MEM_STATS_TYPE_FREE: type = "FREE"; direction = -1; break;
1617 case MEM_STATS_TYPE_MALLOC: type = "MALLOC"; direction = 1; break;
Willy Tarreaua6026a02020-07-02 09:14:48 +02001618 case MEM_STATS_TYPE_REALLOC: type = "REALLOC"; break;
Willy Tarreaub59e3f62022-11-30 17:16:51 +01001619 case MEM_STATS_TYPE_STRDUP: type = "STRDUP"; direction = 1; break;
1620 case MEM_STATS_TYPE_P_ALLOC: type = "P_ALLOC"; direction = 1; if (ptr->extra) info = ((const struct pool_head *)ptr->extra)->name; break;
1621 case MEM_STATS_TYPE_P_FREE: type = "P_FREE"; direction = -1; if (ptr->extra) info = ((const struct pool_head *)ptr->extra)->name; break;
Willy Tarreaua6026a02020-07-02 09:14:48 +02001622 default: type = "UNSET"; break;
1623 }
1624
1625 //chunk_printf(&trash,
1626 // "%20s:%-5d %7s size: %12lu calls: %9lu size/call: %6lu\n",
1627 // name, ptr->line, type,
1628 // (unsigned long)ptr->size, (unsigned long)ptr->calls,
1629 // (unsigned long)(ptr->calls ? (ptr->size / ptr->calls) : 0));
1630
Willy Tarreaue57fbed2022-11-30 16:42:54 +01001631 /* only match requested prefixes */
1632 if (pfx && (!info || strncmp(info, pfx, strlen(pfx)) != 0))
1633 continue;
1634
Willy Tarreau17200dd2022-08-09 08:40:08 +02001635 chunk_reset(&trash);
1636 if (ctx->show_all)
1637 chunk_appendf(&trash, "%s(", func);
1638
Willy Tarreaud96d2142022-09-06 08:05:59 +02001639 chunk_appendf(&trash, "%s:%d", name, ptr->caller.line);
Willy Tarreau17200dd2022-08-09 08:40:08 +02001640
1641 if (ctx->show_all)
1642 chunk_appendf(&trash, ")");
1643
Willy Tarreau4a426e22022-08-09 08:51:08 +02001644 while (trash.data < ctx->width)
Willy Tarreaua6026a02020-07-02 09:14:48 +02001645 trash.area[trash.data++] = ' ';
Willy Tarreau17200dd2022-08-09 08:40:08 +02001646
Willy Tarreau55c950b2022-08-09 08:15:27 +02001647 chunk_appendf(&trash, "%7s size: %12lu calls: %9lu size/call: %6lu %s\n",
Willy Tarreaua6026a02020-07-02 09:14:48 +02001648 type,
1649 (unsigned long)ptr->size, (unsigned long)ptr->calls,
Willy Tarreau55c950b2022-08-09 08:15:27 +02001650 (unsigned long)(ptr->calls ? (ptr->size / ptr->calls) : 0),
1651 info ? info : "");
Willy Tarreaua6026a02020-07-02 09:14:48 +02001652
Willy Tarreaud0a06d52022-05-18 15:07:19 +02001653 if (applet_putchk(appctx, &trash) == -1) {
Willy Tarreau40e952f2022-05-05 14:39:51 +02001654 ctx->start = ptr;
Willy Tarreaua6026a02020-07-02 09:14:48 +02001655 ret = 0;
Willy Tarreaub59e3f62022-11-30 17:16:51 +01001656 goto end;
1657 }
1658 if (direction > 0) {
1659 ctx->tot_size += (ulong)ptr->size;
1660 ctx->tot_calls += (ulong)ptr->calls;
1661 }
1662 else if (direction < 0) {
1663 ctx->tot_size -= (ulong)ptr->size;
1664 ctx->tot_calls += (ulong)ptr->calls;
Willy Tarreaua6026a02020-07-02 09:14:48 +02001665 }
1666 }
1667
Willy Tarreaub59e3f62022-11-30 17:16:51 +01001668 /* now dump a summary */
1669 chunk_reset(&trash);
1670 chunk_appendf(&trash, "Total");
1671 while (trash.data < ctx->width)
1672 trash.area[trash.data++] = ' ';
1673
1674 chunk_appendf(&trash, "%7s size: %12ld calls: %9lu size/call: %6ld %s\n",
1675 "BALANCE",
1676 ctx->tot_size, ctx->tot_calls,
1677 (long)(ctx->tot_calls ? (ctx->tot_size / ctx->tot_calls) : 0),
1678 "(excl. realloc)");
1679
1680 if (applet_putchk(appctx, &trash) == -1) {
1681 ctx->start = ptr;
1682 ret = 0;
1683 goto end;
1684 }
Willy Tarreaua6026a02020-07-02 09:14:48 +02001685 end:
1686 return ret;
1687}
1688
Willy Tarreaue57fbed2022-11-30 16:42:54 +01001689/* release the "show pools" context */
1690static void debug_release_memstats(struct appctx *appctx)
1691{
1692 struct dev_mem_ctx *ctx = appctx->svcctx;
1693
1694 ha_free(&ctx->match);
1695}
Willy Tarreaua6026a02020-07-02 09:14:48 +02001696#endif
1697
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +02001698#ifdef USE_THREAD_DUMP
Willy Tarreauc7091d82019-05-17 10:08:49 +02001699
Willy Tarreauf0c86dd2022-07-15 13:14:03 +02001700/* handles DEBUGSIG to dump the state of the thread it's working on. This is
1701 * appended at the end of thread_dump_buffer which must be protected against
1702 * reentrance from different threads (a thread-local buffer works fine).
1703 */
Willy Tarreauc7091d82019-05-17 10:08:49 +02001704void debug_handler(int sig, siginfo_t *si, void *arg)
1705{
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +02001706 struct buffer *buf = HA_ATOMIC_LOAD(&th_ctx->thread_dump_buffer);
Willy Tarreaua2b8ed42022-07-01 17:11:03 +02001707 int harmless = is_thread_harmless();
1708
Willy Tarreau82aafc42020-03-03 08:31:34 +01001709 /* first, let's check it's really for us and that we didn't just get
1710 * a spurious DEBUGSIG.
1711 */
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +02001712 if (!buf || buf == (void*)(0x1UL))
Willy Tarreau82aafc42020-03-03 08:31:34 +01001713 return;
1714
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +02001715 /* now dump the current state into the designated buffer, and indicate
1716 * we come from a sig handler.
Willy Tarreauf0c86dd2022-07-15 13:14:03 +02001717 */
Willy Tarreau9a6ecbd2023-05-04 16:23:51 +02001718 ha_thread_dump_one(tid, 1);
Willy Tarreaue6a02fa2019-05-22 07:06:44 +02001719
1720 /* mark the current thread as stuck to detect it upon next invocation
1721 * if it didn't move.
1722 */
Willy Tarreaua2b8ed42022-07-01 17:11:03 +02001723 if (!harmless &&
Willy Tarreaue7475c82022-06-20 09:23:24 +02001724 !(_HA_ATOMIC_LOAD(&th_ctx->flags) & TH_FL_SLEEPING))
Willy Tarreaubdcd3252022-06-22 09:19:46 +02001725 _HA_ATOMIC_OR(&th_ctx->flags, TH_FL_STUCK);
Willy Tarreauc7091d82019-05-17 10:08:49 +02001726}
1727
1728static int init_debug_per_thread()
1729{
1730 sigset_t set;
1731
1732 /* unblock the DEBUGSIG signal we intend to use */
1733 sigemptyset(&set);
1734 sigaddset(&set, DEBUGSIG);
Willy Tarreaubd3b44e2023-03-09 08:25:01 +01001735#if defined(DEBUG_DEV)
1736 sigaddset(&set, SIGRTMAX);
1737#endif
Willy Tarreauc7091d82019-05-17 10:08:49 +02001738 ha_sigmask(SIG_UNBLOCK, &set, NULL);
1739 return 1;
1740}
1741
1742static int init_debug()
1743{
1744 struct sigaction sa;
Willy Tarreau2f1227e2021-01-22 12:12:29 +01001745 void *callers[1];
Willy Tarreauc7091d82019-05-17 10:08:49 +02001746
Willy Tarreau0214b452020-03-04 06:01:40 +01001747 /* calling backtrace() will access libgcc at runtime. We don't want to
1748 * do it after the chroot, so let's perform a first call to have it
1749 * ready in memory for later use.
1750 */
Willy Tarreau13faf162020-03-04 07:44:06 +01001751 my_backtrace(callers, sizeof(callers)/sizeof(*callers));
Willy Tarreauc7091d82019-05-17 10:08:49 +02001752 sa.sa_handler = NULL;
1753 sa.sa_sigaction = debug_handler;
1754 sigemptyset(&sa.sa_mask);
1755 sa.sa_flags = SA_SIGINFO;
1756 sigaction(DEBUGSIG, &sa, NULL);
Willy Tarreaubd3b44e2023-03-09 08:25:01 +01001757
1758#if defined(DEBUG_DEV)
1759 sa.sa_handler = NULL;
1760 sa.sa_sigaction = debug_delay_inj_sighandler;
1761 sigemptyset(&sa.sa_mask);
1762 sa.sa_flags = SA_SIGINFO;
1763 sigaction(SIGRTMAX, &sa, NULL);
1764#endif
Christopher Fauletfc633b62020-11-06 15:24:23 +01001765 return ERR_NONE;
Willy Tarreauc7091d82019-05-17 10:08:49 +02001766}
1767
1768REGISTER_POST_CHECK(init_debug);
1769REGISTER_PER_THREAD_INIT(init_debug_per_thread);
1770
1771#endif /* USE_THREAD_DUMP */
1772
Willy Tarreau4e2b6462019-05-16 17:44:30 +02001773/* register cli keywords */
1774static struct cli_kw_list cli_kws = {{ },{
Willy Tarreaub205bfd2021-05-07 11:38:37 +02001775 {{ "debug", "dev", "bug", NULL }, "debug dev bug : call BUG_ON() and crash", debug_parse_cli_bug, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreau6d3f1e32022-02-28 11:51:23 +01001776 {{ "debug", "dev", "check", NULL }, "debug dev check : call CHECK_IF() and possibly crash", debug_parse_cli_check, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreaub205bfd2021-05-07 11:38:37 +02001777 {{ "debug", "dev", "close", NULL }, "debug dev close <fd> : close this file descriptor", debug_parse_cli_close, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreaudadf00e2022-07-15 08:25:03 +02001778 {{ "debug", "dev", "deadlock", NULL }, "debug dev deadlock [nbtask] : deadlock between this number of tasks", debug_parse_cli_deadlock, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreaub205bfd2021-05-07 11:38:37 +02001779 {{ "debug", "dev", "delay", NULL }, "debug dev delay [ms] : sleep this long", debug_parse_cli_delay, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreau6bdf3e92019-05-20 14:25:05 +02001780#if defined(DEBUG_DEV)
Willy Tarreaubd3b44e2023-03-09 08:25:01 +01001781 {{ "debug", "dev", "delay-inj", NULL },"debug dev delay-inj <inter> <count> : inject random delays into threads", debug_parse_delay_inj, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreaub205bfd2021-05-07 11:38:37 +02001782 {{ "debug", "dev", "exec", NULL }, "debug dev exec [cmd] ... : show this command's output", debug_parse_cli_exec, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreau6bdf3e92019-05-20 14:25:05 +02001783#endif
Willy Tarreau5be7c192022-01-24 20:26:09 +01001784 {{ "debug", "dev", "fd", NULL }, "debug dev fd : scan for rogue/unhandled FDs", debug_parse_cli_fd, debug_iohandler_fd, NULL, NULL, ACCESS_EXPERT },
Willy Tarreaub205bfd2021-05-07 11:38:37 +02001785 {{ "debug", "dev", "exit", NULL }, "debug dev exit [code] : immediately exit the process", debug_parse_cli_exit, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreau111c7832022-11-30 17:51:47 +01001786 {{ "debug", "dev", "hash", NULL }, "debug dev hash [msg] : return msg hashed if anon is set", debug_parse_cli_hash, NULL, NULL, NULL, 0 },
Willy Tarreaub205bfd2021-05-07 11:38:37 +02001787 {{ "debug", "dev", "hex", NULL }, "debug dev hex <addr> [len] : dump a memory area", debug_parse_cli_hex, NULL, NULL, NULL, ACCESS_EXPERT },
1788 {{ "debug", "dev", "log", NULL }, "debug dev log [msg] ... : send this msg to global logs", debug_parse_cli_log, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreaue5e62232023-05-04 11:50:26 +02001789 {{ "debug", "dev", "loop", NULL }, "debug dev loop <ms> [isolated] : loop this long, possibly isolated", debug_parse_cli_loop, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreaua6026a02020-07-02 09:14:48 +02001790#if defined(DEBUG_MEM_STATS)
Willy Tarreaue57fbed2022-11-30 16:42:54 +01001791 {{ "debug", "dev", "memstats", NULL }, "debug dev memstats [reset|all|match ...]: dump/reset memory statistics", debug_parse_cli_memstats, debug_iohandler_memstats, debug_release_memstats, NULL, 0 },
Willy Tarreaua6026a02020-07-02 09:14:48 +02001792#endif
Willy Tarreaub205bfd2021-05-07 11:38:37 +02001793 {{ "debug", "dev", "panic", NULL }, "debug dev panic : immediately trigger a panic", debug_parse_cli_panic, NULL, NULL, NULL, ACCESS_EXPERT },
1794 {{ "debug", "dev", "sched", NULL }, "debug dev sched {task|tasklet} [k=v]* : stress the scheduler", debug_parse_cli_sched, NULL, NULL, NULL, ACCESS_EXPERT },
1795 {{ "debug", "dev", "stream",NULL }, "debug dev stream [k=v]* : show/manipulate stream flags", debug_parse_cli_stream,NULL, NULL, NULL, ACCESS_EXPERT },
1796 {{ "debug", "dev", "sym", NULL }, "debug dev sym <addr> : resolve symbol address", debug_parse_cli_sym, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreau98679872023-05-03 11:22:45 +02001797 {{ "debug", "dev", "task", NULL }, "debug dev task <ptr> [wake|expire|kill] : show/wake/expire/kill task/tasklet", debug_parse_cli_task, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreaub205bfd2021-05-07 11:38:37 +02001798 {{ "debug", "dev", "tkill", NULL }, "debug dev tkill [thr] [sig] : send signal to thread", debug_parse_cli_tkill, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreau305cfbd2022-02-25 08:52:39 +01001799 {{ "debug", "dev", "warn", NULL }, "debug dev warn : call WARN_ON() and possibly crash", debug_parse_cli_warn, NULL, NULL, NULL, ACCESS_EXPERT },
Willy Tarreaub205bfd2021-05-07 11:38:37 +02001800 {{ "debug", "dev", "write", NULL }, "debug dev write [size] : write that many bytes in return", debug_parse_cli_write, NULL, NULL, NULL, ACCESS_EXPERT },
Erwan Le Goas54966df2022-09-14 17:24:22 +02001801
Willy Tarreau6ab7b212021-12-28 09:57:10 +01001802#if defined(HA_HAVE_DUMP_LIBS)
1803 {{ "show", "libs", NULL, NULL }, "show libs : show loaded object files and libraries", debug_parse_cli_show_libs, NULL, NULL },
1804#endif
Willy Tarreaub205bfd2021-05-07 11:38:37 +02001805 {{ "show", "threads", NULL, NULL }, "show threads : show some threads debugging information", NULL, cli_io_handler_show_threads, NULL },
Willy Tarreau4e2b6462019-05-16 17:44:30 +02001806 {{},}
1807}};
1808
1809INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);