MINOR: debug: add random delay injection with "debug dev delay-inj"
The goal is to send signals to random threads at random instants so that
they spin for a random delay in a relax() loop, trying to give back the
CPU to another competing hardware thread, in hope that from time to time
this can trigger in critical areas and increase the chances to provoke a
latent concurrency bug. For now none were observed.
For example, this command starts 64 such tasks waking after random delays
of 0-1ms and delivering signals to trigger such loops on 3 random threads:
for i in {1..64}; do
socat - /tmp/sock1 <<< "expert-mode on;debug dev delay-inj 2 3"
done
This command is only enabled when DEBUG_DEV is set at build time.
diff --git a/src/debug.c b/src/debug.c
index 9f9a87d..3d159fd 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -624,6 +624,18 @@
thread_release();
return cli_err(appctx, "Failed to execute command.\n");
}
+
+/* handles SIGRTMAX to inject random delays on the receiving thread in order
+ * to try to increase the likelihood to reproduce inter-thread races. The
+ * signal is periodically sent by a task initiated by "debug dev delay-inj".
+ */
+void debug_delay_inj_sighandler(int sig, siginfo_t *si, void *arg)
+{
+ volatile int i = statistical_prng_range(10000);
+
+ while (i--)
+ __ha_cpu_relax();
+}
#endif
/* parse a "debug dev hex" command. It always returns 1. */
@@ -872,6 +884,65 @@
return 1;
}
+#if defined(DEBUG_DEV)
+static struct task *debug_delay_inj_task(struct task *t, void *ctx, unsigned int state)
+{
+ unsigned long *tctx = ctx; // [0] = interval, [1] = nbwakeups
+ unsigned long inter = tctx[0];
+ unsigned long count = tctx[1];
+ unsigned long rnd;
+
+ if (inter)
+ t->expire = tick_add(now_ms, inter);
+ else
+ task_wakeup(t, TASK_WOKEN_MSG);
+
+ /* wake a random thread */
+ while (count--) {
+ rnd = statistical_prng_range(global.nbthread);
+ ha_tkill(rnd, SIGRTMAX);
+ }
+ return t;
+}
+
+/* parse a "debug dev delay-inj" command
+ * debug dev delay-inj <inter> <count>
+ */
+static int debug_parse_delay_inj(char **args, char *payload, struct appctx *appctx, void *private)
+{
+ unsigned long *tctx; // [0] = inter, [2] = count
+ struct task *task;
+
+ if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
+ return 1;
+
+ if (!*args[4])
+ return cli_err(appctx, "Usage: debug dev delay-inj <inter_ms> <count>*\n");
+
+ _HA_ATOMIC_INC(&debug_commands_issued);
+
+ tctx = calloc(sizeof(*tctx), 2);
+ if (!tctx)
+ goto fail;
+
+ tctx[0] = atoi(args[3]);
+ tctx[1] = atoi(args[4]);
+
+ task = task_new_here/*anywhere*/();
+ if (!task)
+ goto fail;
+
+ task->process = debug_delay_inj_task;
+ task->context = tctx;
+ task_wakeup(task, TASK_WOKEN_INIT);
+ return 1;
+
+ fail:
+ free(tctx);
+ return cli_err(appctx, "Not enough memory");
+}
+#endif // DEBUG_DEV
+
static struct task *debug_task_handler(struct task *t, void *ctx, unsigned int state)
{
unsigned long *tctx = ctx; // [0] = #tasks, [1] = inter, [2+] = { tl | (tsk+1) }
@@ -1623,6 +1694,9 @@
/* unblock the DEBUGSIG signal we intend to use */
sigemptyset(&set);
sigaddset(&set, DEBUGSIG);
+#if defined(DEBUG_DEV)
+ sigaddset(&set, SIGRTMAX);
+#endif
ha_sigmask(SIG_UNBLOCK, &set, NULL);
return 1;
}
@@ -1642,6 +1716,14 @@
sigemptyset(&sa.sa_mask);
sa.sa_flags = SA_SIGINFO;
sigaction(DEBUGSIG, &sa, NULL);
+
+#if defined(DEBUG_DEV)
+ sa.sa_handler = NULL;
+ sa.sa_sigaction = debug_delay_inj_sighandler;
+ sigemptyset(&sa.sa_mask);
+ sa.sa_flags = SA_SIGINFO;
+ sigaction(SIGRTMAX, &sa, NULL);
+#endif
return ERR_NONE;
}
@@ -1658,6 +1740,7 @@
{{ "debug", "dev", "deadlock", NULL }, "debug dev deadlock [nbtask] : deadlock between this number of tasks", debug_parse_cli_deadlock, NULL, NULL, NULL, ACCESS_EXPERT },
{{ "debug", "dev", "delay", NULL }, "debug dev delay [ms] : sleep this long", debug_parse_cli_delay, NULL, NULL, NULL, ACCESS_EXPERT },
#if defined(DEBUG_DEV)
+ {{ "debug", "dev", "delay-inj", NULL },"debug dev delay-inj <inter> <count> : inject random delays into threads", debug_parse_delay_inj, NULL, NULL, NULL, ACCESS_EXPERT },
{{ "debug", "dev", "exec", NULL }, "debug dev exec [cmd] ... : show this command's output", debug_parse_cli_exec, NULL, NULL, NULL, ACCESS_EXPERT },
#endif
{{ "debug", "dev", "fd", NULL }, "debug dev fd : scan for rogue/unhandled FDs", debug_parse_cli_fd, debug_iohandler_fd, NULL, NULL, ACCESS_EXPERT },