blob: 4a7def1bebedebdbc1ae849de6b3c55452449b50 [file] [log] [blame]
Willy Tarreau609aad92018-11-22 08:31:09 +01001/*
2 * activity measurement functions.
3 *
4 * Copyright 2000-2018 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
Willy Tarreaub2551052020-06-09 09:07:15 +020013#include <haproxy/activity-t.h>
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020014#include <haproxy/api.h>
Willy Tarreau6be78492020-06-05 00:00:29 +020015#include <haproxy/cfgparse.h>
Willy Tarreauf1d32c42020-06-04 21:07:02 +020016#include <haproxy/channel.h>
Willy Tarreau83487a82020-06-04 20:19:54 +020017#include <haproxy/cli.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020018#include <haproxy/freq_ctr.h>
Willy Tarreau5e539c92020-06-04 20:45:39 +020019#include <haproxy/stream_interface.h>
Willy Tarreau48fbcae2020-06-03 18:09:46 +020020#include <haproxy/tools.h>
Willy Tarreau75c62c22018-11-22 11:02:09 +010021
Willy Tarreauf93c7be2021-05-05 17:07:09 +020022#if defined(DEBUG_MEM_STATS)
23/* these ones are macros in bug.h when DEBUG_MEM_STATS is set, and will
24 * prevent the new ones from being redefined.
25 */
26#undef calloc
27#undef malloc
28#undef realloc
29#endif
Willy Tarreau75c62c22018-11-22 11:02:09 +010030
31/* bit field of profiling options. Beware, may be modified at runtime! */
Willy Tarreauef7380f2021-05-05 16:28:31 +020032unsigned int profiling __read_mostly = HA_PROF_TASKS_AOFF;
33unsigned long task_profiling_mask __read_mostly = 0;
Willy Tarreau609aad92018-11-22 08:31:09 +010034
35/* One struct per thread containing all collected measurements */
36struct activity activity[MAX_THREADS] __attribute__((aligned(64))) = { };
37
Willy Tarreau3fb6a7b2021-01-28 19:19:26 +010038/* One struct per function pointer hash entry (256 values, 0=collision) */
39struct sched_activity sched_activity[256] __attribute__((aligned(64))) = { };
Willy Tarreau609aad92018-11-22 08:31:09 +010040
Willy Tarreaudb87fc72021-05-05 16:50:40 +020041
Willy Tarreau3e109ee2021-08-28 12:04:25 +020042#ifdef USE_MEMORY_PROFILING
Willy Tarreaudb87fc72021-05-05 16:50:40 +020043/* determine the number of buckets to store stats */
44#define MEMPROF_HASH_BITS 10
45#define MEMPROF_HASH_BUCKETS (1U << MEMPROF_HASH_BITS)
46
Willy Tarreau616491b2021-05-11 09:26:23 +020047enum memprof_method {
48 MEMPROF_METH_UNKNOWN = 0,
49 MEMPROF_METH_MALLOC,
50 MEMPROF_METH_CALLOC,
51 MEMPROF_METH_REALLOC,
52 MEMPROF_METH_FREE,
53 MEMPROF_METH_METHODS /* count, must be last */
54};
55
56static const char *const memprof_methods[MEMPROF_METH_METHODS] = {
57 "unknown", "malloc", "calloc", "realloc", "free",
58};
59
Willy Tarreaudb87fc72021-05-05 16:50:40 +020060/* stats:
61 * - malloc increases alloc
62 * - free increases free (if non null)
63 * - realloc increases either depending on the size change.
64 * when the real size is known (malloc_usable_size()), it's used in free_tot
65 * and alloc_tot, otherwise the requested size is reported in alloc_tot and
66 * zero in free_tot.
67 */
68struct memprof_stats {
69 const void *caller;
Willy Tarreau616491b2021-05-11 09:26:23 +020070 enum memprof_method method;
71 /* 4-7 bytes hole here */
Willy Tarreaudb87fc72021-05-05 16:50:40 +020072 unsigned long long alloc_calls;
73 unsigned long long free_calls;
74 unsigned long long alloc_tot;
75 unsigned long long free_tot;
76};
77
78/* last one is for hash collisions ("others") and has no caller address */
79struct memprof_stats memprof_stats[MEMPROF_HASH_BUCKETS + 1] = { };
80
Willy Tarreauf93c7be2021-05-05 17:07:09 +020081/* used to detect recursive calls */
82static THREAD_LOCAL int in_memprof = 0;
83
84/* perform a pointer hash by scrambling its bits and retrieving the most
85 * mixed ones (topmost ones in 32-bit, middle ones in 64-bit).
86 */
87static unsigned int memprof_hash_ptr(const void *p)
88{
89 unsigned long long x = (unsigned long)p;
90
91 x = 0xcbda9653U * x;
92 if (sizeof(long) == 4)
93 x >>= 32;
94 else
95 x >>= 33 - MEMPROF_HASH_BITS / 2;
96 return x & (MEMPROF_HASH_BUCKETS - 1);
97}
98
99/* These ones are used by glibc and will be called early. They are in charge of
100 * initializing the handlers with the original functions.
101 */
102static void *memprof_malloc_initial_handler(size_t size);
103static void *memprof_calloc_initial_handler(size_t nmemb, size_t size);
104static void *memprof_realloc_initial_handler(void *ptr, size_t size);
105static void memprof_free_initial_handler(void *ptr);
106
107/* Fallback handlers for the main alloc/free functions. They are preset to
108 * the initializer in order to save a test in the functions's critical path.
109 */
110static void *(*memprof_malloc_handler)(size_t size) = memprof_malloc_initial_handler;
111static void *(*memprof_calloc_handler)(size_t nmemb, size_t size) = memprof_calloc_initial_handler;
112static void *(*memprof_realloc_handler)(void *ptr, size_t size) = memprof_realloc_initial_handler;
113static void (*memprof_free_handler)(void *ptr) = memprof_free_initial_handler;
114
115/* Used to force to die if it's not possible to retrieve the allocation
116 * functions. We cannot even use stdio in this case.
117 */
118static __attribute__((noreturn)) void memprof_die(const char *msg)
119{
120 DISGUISE(write(2, msg, strlen(msg)));
121 exit(1);
122}
123
124/* Resolve original allocation functions and initialize all handlers.
125 * This must be called very early at boot, before the very first malloc()
126 * call, and is not thread-safe! It's not even possible to use stdio there.
127 * Worse, we have to account for the risk of reentrance from dlsym() when
128 * it tries to prepare its error messages. Here its ahndled by in_memprof
129 * that makes allocators return NULL. dlsym() handles it gracefully. An
Ilya Shipitsin3df59892021-05-10 12:50:00 +0500130 * alternate approach consists in calling aligned_alloc() from these places
Willy Tarreauf93c7be2021-05-05 17:07:09 +0200131 * but that would mean not being able to intercept it later if considered
132 * useful to do so.
133 */
134static void memprof_init()
135{
136 in_memprof++;
137 memprof_malloc_handler = get_sym_next_addr("malloc");
138 if (!memprof_malloc_handler)
139 memprof_die("FATAL: malloc() function not found.\n");
140
141 memprof_calloc_handler = get_sym_next_addr("calloc");
142 if (!memprof_calloc_handler)
143 memprof_die("FATAL: calloc() function not found.\n");
144
145 memprof_realloc_handler = get_sym_next_addr("realloc");
146 if (!memprof_realloc_handler)
147 memprof_die("FATAL: realloc() function not found.\n");
148
149 memprof_free_handler = get_sym_next_addr("free");
150 if (!memprof_free_handler)
151 memprof_die("FATAL: free() function not found.\n");
152 in_memprof--;
153}
154
155/* the initial handlers will initialize all regular handlers and will call the
156 * one they correspond to. A single one of these functions will typically be
157 * called, though it's unknown which one (as any might be called before main).
158 */
159static void *memprof_malloc_initial_handler(size_t size)
160{
161 if (in_memprof) {
162 /* it's likely that dlsym() needs malloc(), let's fail */
163 return NULL;
164 }
165
166 memprof_init();
167 return memprof_malloc_handler(size);
168}
169
170static void *memprof_calloc_initial_handler(size_t nmemb, size_t size)
171{
172 if (in_memprof) {
173 /* it's likely that dlsym() needs calloc(), let's fail */
174 return NULL;
175 }
176 memprof_init();
177 return memprof_calloc_handler(nmemb, size);
178}
179
180static void *memprof_realloc_initial_handler(void *ptr, size_t size)
181{
182 if (in_memprof) {
183 /* it's likely that dlsym() needs realloc(), let's fail */
184 return NULL;
185 }
186
187 memprof_init();
188 return memprof_realloc_handler(ptr, size);
189}
190
191static void memprof_free_initial_handler(void *ptr)
192{
193 memprof_init();
194 memprof_free_handler(ptr);
195}
196
197/* Assign a bin for the memprof_stats to the return address. May perform a few
198 * attempts before finding the right one, but always succeeds (in the worst
199 * case, returns a default bin). The caller address is atomically set except
200 * for the default one which is never set.
201 */
Willy Tarreau616491b2021-05-11 09:26:23 +0200202static struct memprof_stats *memprof_get_bin(const void *ra, enum memprof_method meth)
Willy Tarreauf93c7be2021-05-05 17:07:09 +0200203{
204 int retries = 16; // up to 16 consecutive entries may be tested.
Willy Tarreau4a753282021-05-09 23:18:50 +0200205 const void *old;
Willy Tarreauf93c7be2021-05-05 17:07:09 +0200206 unsigned int bin;
207
208 bin = memprof_hash_ptr(ra);
209 for (; memprof_stats[bin].caller != ra; bin = (bin + 1) & (MEMPROF_HASH_BUCKETS - 1)) {
210 if (!--retries) {
211 bin = MEMPROF_HASH_BUCKETS;
212 break;
213 }
214
215 old = NULL;
216 if (!memprof_stats[bin].caller &&
Willy Tarreau616491b2021-05-11 09:26:23 +0200217 HA_ATOMIC_CAS(&memprof_stats[bin].caller, &old, ra)) {
218 memprof_stats[bin].method = meth;
Willy Tarreauf93c7be2021-05-05 17:07:09 +0200219 break;
Willy Tarreau616491b2021-05-11 09:26:23 +0200220 }
Willy Tarreauf93c7be2021-05-05 17:07:09 +0200221 }
222 return &memprof_stats[bin];
223}
224
225/* This is the new global malloc() function. It must optimize for the normal
226 * case (i.e. profiling disabled) hence the first test to permit a direct jump.
227 * It must remain simple to guarantee the lack of reentrance. stdio is not
228 * possible there even for debugging. The reported size is the really allocated
229 * one as returned by malloc_usable_size(), because this will allow it to be
230 * compared to the one before realloc() or free(). This is a GNU and jemalloc
231 * extension but other systems may also store this size in ptr[-1].
232 */
233void *malloc(size_t size)
234{
235 struct memprof_stats *bin;
236 void *ret;
237
238 if (likely(!(profiling & HA_PROF_MEMORY)))
239 return memprof_malloc_handler(size);
240
241 ret = memprof_malloc_handler(size);
242 size = malloc_usable_size(ret);
243
Willy Tarreau616491b2021-05-11 09:26:23 +0200244 bin = memprof_get_bin(__builtin_return_address(0), MEMPROF_METH_MALLOC);
Willy Tarreauf93c7be2021-05-05 17:07:09 +0200245 _HA_ATOMIC_ADD(&bin->alloc_calls, 1);
246 _HA_ATOMIC_ADD(&bin->alloc_tot, size);
247 return ret;
248}
249
250/* This is the new global calloc() function. It must optimize for the normal
251 * case (i.e. profiling disabled) hence the first test to permit a direct jump.
252 * It must remain simple to guarantee the lack of reentrance. stdio is not
253 * possible there even for debugging. The reported size is the really allocated
254 * one as returned by malloc_usable_size(), because this will allow it to be
255 * compared to the one before realloc() or free(). This is a GNU and jemalloc
256 * extension but other systems may also store this size in ptr[-1].
257 */
258void *calloc(size_t nmemb, size_t size)
259{
260 struct memprof_stats *bin;
261 void *ret;
262
263 if (likely(!(profiling & HA_PROF_MEMORY)))
264 return memprof_calloc_handler(nmemb, size);
265
266 ret = memprof_calloc_handler(nmemb, size);
267 size = malloc_usable_size(ret);
268
Willy Tarreau616491b2021-05-11 09:26:23 +0200269 bin = memprof_get_bin(__builtin_return_address(0), MEMPROF_METH_CALLOC);
Willy Tarreauf93c7be2021-05-05 17:07:09 +0200270 _HA_ATOMIC_ADD(&bin->alloc_calls, 1);
271 _HA_ATOMIC_ADD(&bin->alloc_tot, size);
272 return ret;
273}
274
275/* This is the new global realloc() function. It must optimize for the normal
276 * case (i.e. profiling disabled) hence the first test to permit a direct jump.
277 * It must remain simple to guarantee the lack of reentrance. stdio is not
278 * possible there even for debugging. The reported size is the really allocated
279 * one as returned by malloc_usable_size(), because this will allow it to be
280 * compared to the one before realloc() or free(). This is a GNU and jemalloc
281 * extension but other systems may also store this size in ptr[-1].
282 * Depending on the old vs new size, it's considered as an allocation or a free
283 * (or neither if the size remains the same).
284 */
285void *realloc(void *ptr, size_t size)
286{
287 struct memprof_stats *bin;
288 size_t size_before;
289 void *ret;
290
291 if (likely(!(profiling & HA_PROF_MEMORY)))
292 return memprof_realloc_handler(ptr, size);
293
294 size_before = malloc_usable_size(ptr);
295 ret = memprof_realloc_handler(ptr, size);
Willy Tarreau2639e2e2021-05-07 08:01:35 +0200296 size = malloc_usable_size(ret);
Willy Tarreauf93c7be2021-05-05 17:07:09 +0200297
Willy Tarreau616491b2021-05-11 09:26:23 +0200298 bin = memprof_get_bin(__builtin_return_address(0), MEMPROF_METH_REALLOC);
Willy Tarreauf93c7be2021-05-05 17:07:09 +0200299 if (size > size_before) {
300 _HA_ATOMIC_ADD(&bin->alloc_calls, 1);
Willy Tarreau79acefa2021-05-11 09:12:56 +0200301 _HA_ATOMIC_ADD(&bin->alloc_tot, size - size_before);
Willy Tarreauf93c7be2021-05-05 17:07:09 +0200302 } else if (size < size_before) {
303 _HA_ATOMIC_ADD(&bin->free_calls, 1);
Willy Tarreau79acefa2021-05-11 09:12:56 +0200304 _HA_ATOMIC_ADD(&bin->free_tot, size_before - size);
Willy Tarreauf93c7be2021-05-05 17:07:09 +0200305 }
306 return ret;
307}
308
309/* This is the new global free() function. It must optimize for the normal
310 * case (i.e. profiling disabled) hence the first test to permit a direct jump.
311 * It must remain simple to guarantee the lack of reentrance. stdio is not
312 * possible there even for debugging. The reported size is the really allocated
313 * one as returned by malloc_usable_size(), because this will allow it to be
314 * compared to the one before realloc() or free(). This is a GNU and jemalloc
315 * extension but other systems may also store this size in ptr[-1]. Since
316 * free() is often called on NULL pointers to collect garbage at the end of
317 * many functions or during config parsing, as a special case free(NULL)
318 * doesn't update any stats.
319 */
320void free(void *ptr)
321{
322 struct memprof_stats *bin;
323 size_t size_before;
324
325 if (likely(!(profiling & HA_PROF_MEMORY) || !ptr)) {
326 memprof_free_handler(ptr);
327 return;
328 }
329
330 size_before = malloc_usable_size(ptr);
331 memprof_free_handler(ptr);
332
Willy Tarreau616491b2021-05-11 09:26:23 +0200333 bin = memprof_get_bin(__builtin_return_address(0), MEMPROF_METH_FREE);
Willy Tarreauf93c7be2021-05-05 17:07:09 +0200334 _HA_ATOMIC_ADD(&bin->free_calls, 1);
335 _HA_ATOMIC_ADD(&bin->free_tot, size_before);
336}
337
Willy Tarreaudb87fc72021-05-05 16:50:40 +0200338#endif // USE_MEMORY_PROFILING
339
Willy Tarreau609aad92018-11-22 08:31:09 +0100340/* Updates the current thread's statistics about stolen CPU time. The unit for
341 * <stolen> is half-milliseconds.
342 */
343void report_stolen_time(uint64_t stolen)
344{
345 activity[tid].cpust_total += stolen;
346 update_freq_ctr(&activity[tid].cpust_1s, stolen);
347 update_freq_ctr_period(&activity[tid].cpust_15s, 15000, stolen);
348}
Willy Tarreau75c62c22018-11-22 11:02:09 +0100349
Willy Tarreauca3afc22021-05-05 18:33:19 +0200350#ifdef USE_MEMORY_PROFILING
351/* config parser for global "profiling.memory", accepts "on" or "off" */
352static int cfg_parse_prof_memory(char **args, int section_type, struct proxy *curpx,
353 const struct proxy *defpx, const char *file, int line,
354 char **err)
355{
356 if (too_many_args(1, args, err, NULL))
357 return -1;
358
359 if (strcmp(args[1], "on") == 0)
360 profiling |= HA_PROF_MEMORY;
361 else if (strcmp(args[1], "off") == 0)
362 profiling &= ~HA_PROF_MEMORY;
363 else {
364 memprintf(err, "'%s' expects either 'on' or 'off' but got '%s'.", args[0], args[1]);
365 return -1;
366 }
367 return 0;
368}
369#endif // USE_MEMORY_PROFILING
370
Willy Tarreau75c62c22018-11-22 11:02:09 +0100371/* config parser for global "profiling.tasks", accepts "on" or "off" */
372static int cfg_parse_prof_tasks(char **args, int section_type, struct proxy *curpx,
Willy Tarreau01825162021-03-09 09:53:46 +0100373 const struct proxy *defpx, const char *file, int line,
Willy Tarreau75c62c22018-11-22 11:02:09 +0100374 char **err)
375{
376 if (too_many_args(1, args, err, NULL))
377 return -1;
378
379 if (strcmp(args[1], "on") == 0)
Willy Tarreaud2d33482019-04-25 17:09:07 +0200380 profiling = (profiling & ~HA_PROF_TASKS_MASK) | HA_PROF_TASKS_ON;
381 else if (strcmp(args[1], "auto") == 0)
Willy Tarreauaa622b82021-01-28 21:44:22 +0100382 profiling = (profiling & ~HA_PROF_TASKS_MASK) | HA_PROF_TASKS_AOFF;
Willy Tarreau75c62c22018-11-22 11:02:09 +0100383 else if (strcmp(args[1], "off") == 0)
Willy Tarreaud2d33482019-04-25 17:09:07 +0200384 profiling = (profiling & ~HA_PROF_TASKS_MASK) | HA_PROF_TASKS_OFF;
Willy Tarreau75c62c22018-11-22 11:02:09 +0100385 else {
Willy Tarreaud2d33482019-04-25 17:09:07 +0200386 memprintf(err, "'%s' expects either 'on', 'auto', or 'off' but got '%s'.", args[0], args[1]);
Willy Tarreau75c62c22018-11-22 11:02:09 +0100387 return -1;
388 }
389 return 0;
390}
391
392/* parse a "set profiling" command. It always returns 1. */
393static int cli_parse_set_profiling(char **args, char *payload, struct appctx *appctx, void *private)
394{
Willy Tarreau75c62c22018-11-22 11:02:09 +0100395 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
396 return 1;
397
Willy Tarreau00dd44f2021-05-05 16:44:23 +0200398 if (strcmp(args[2], "memory") == 0) {
Willy Tarreaudb87fc72021-05-05 16:50:40 +0200399#ifdef USE_MEMORY_PROFILING
400 if (strcmp(args[3], "on") == 0) {
401 unsigned int old = profiling;
402 int i;
403
404 while (!_HA_ATOMIC_CAS(&profiling, &old, old | HA_PROF_MEMORY))
405 ;
406
407 /* also flush current profiling stats */
408 for (i = 0; i < sizeof(memprof_stats) / sizeof(memprof_stats[0]); i++) {
409 HA_ATOMIC_STORE(&memprof_stats[i].alloc_calls, 0);
410 HA_ATOMIC_STORE(&memprof_stats[i].free_calls, 0);
411 HA_ATOMIC_STORE(&memprof_stats[i].alloc_tot, 0);
412 HA_ATOMIC_STORE(&memprof_stats[i].free_tot, 0);
413 HA_ATOMIC_STORE(&memprof_stats[i].caller, NULL);
414 }
415 }
416 else if (strcmp(args[3], "off") == 0) {
417 unsigned int old = profiling;
418
419 while (!_HA_ATOMIC_CAS(&profiling, &old, old & ~HA_PROF_MEMORY))
420 ;
421 }
422 else
423 return cli_err(appctx, "Expects either 'on' or 'off'.\n");
424 return 1;
425#else
Willy Tarreau00dd44f2021-05-05 16:44:23 +0200426 return cli_err(appctx, "Memory profiling not compiled in.\n");
Willy Tarreaudb87fc72021-05-05 16:50:40 +0200427#endif
Willy Tarreau00dd44f2021-05-05 16:44:23 +0200428 }
429
Willy Tarreau9d008692019-08-09 11:21:01 +0200430 if (strcmp(args[2], "tasks") != 0)
Ilya Shipitsin3df59892021-05-10 12:50:00 +0500431 return cli_err(appctx, "Expects either 'tasks' or 'memory'.\n");
Willy Tarreau75c62c22018-11-22 11:02:09 +0100432
Willy Tarreaud2d33482019-04-25 17:09:07 +0200433 if (strcmp(args[3], "on") == 0) {
434 unsigned int old = profiling;
Willy Tarreaucfa71012021-01-29 11:56:21 +0100435 int i;
436
Willy Tarreaud2d33482019-04-25 17:09:07 +0200437 while (!_HA_ATOMIC_CAS(&profiling, &old, (old & ~HA_PROF_TASKS_MASK) | HA_PROF_TASKS_ON))
438 ;
Willy Tarreaucfa71012021-01-29 11:56:21 +0100439 /* also flush current profiling stats */
440 for (i = 0; i < 256; i++) {
441 HA_ATOMIC_STORE(&sched_activity[i].calls, 0);
442 HA_ATOMIC_STORE(&sched_activity[i].cpu_time, 0);
443 HA_ATOMIC_STORE(&sched_activity[i].lat_time, 0);
444 HA_ATOMIC_STORE(&sched_activity[i].func, NULL);
445 }
Willy Tarreaud2d33482019-04-25 17:09:07 +0200446 }
447 else if (strcmp(args[3], "auto") == 0) {
448 unsigned int old = profiling;
Willy Tarreauaa622b82021-01-28 21:44:22 +0100449 unsigned int new;
450
451 do {
452 if ((old & HA_PROF_TASKS_MASK) >= HA_PROF_TASKS_AON)
453 new = (old & ~HA_PROF_TASKS_MASK) | HA_PROF_TASKS_AON;
454 else
455 new = (old & ~HA_PROF_TASKS_MASK) | HA_PROF_TASKS_AOFF;
456 } while (!_HA_ATOMIC_CAS(&profiling, &old, new));
Willy Tarreaud2d33482019-04-25 17:09:07 +0200457 }
458 else if (strcmp(args[3], "off") == 0) {
459 unsigned int old = profiling;
460 while (!_HA_ATOMIC_CAS(&profiling, &old, (old & ~HA_PROF_TASKS_MASK) | HA_PROF_TASKS_OFF))
461 ;
462 }
Willy Tarreau9d008692019-08-09 11:21:01 +0200463 else
464 return cli_err(appctx, "Expects 'on', 'auto', or 'off'.\n");
465
Willy Tarreau75c62c22018-11-22 11:02:09 +0100466 return 1;
467}
468
Willy Tarreauf1c8a382021-05-13 10:00:17 +0200469static int cmp_sched_activity_calls(const void *a, const void *b)
Willy Tarreau1bd67e92021-01-29 00:07:40 +0100470{
471 const struct sched_activity *l = (const struct sched_activity *)a;
472 const struct sched_activity *r = (const struct sched_activity *)b;
473
474 if (l->calls > r->calls)
475 return -1;
476 else if (l->calls < r->calls)
477 return 1;
478 else
479 return 0;
480}
481
Willy Tarreauf1c8a382021-05-13 10:00:17 +0200482static int cmp_sched_activity_addr(const void *a, const void *b)
483{
484 const struct sched_activity *l = (const struct sched_activity *)a;
485 const struct sched_activity *r = (const struct sched_activity *)b;
486
487 if (l->func > r->func)
488 return -1;
489 else if (l->func < r->func)
490 return 1;
491 else
492 return 0;
493}
494
Willy Tarreau3e109ee2021-08-28 12:04:25 +0200495#ifdef USE_MEMORY_PROFILING
Willy Tarreau993d44d2021-05-05 18:07:02 +0200496/* used by qsort below */
497static int cmp_memprof_stats(const void *a, const void *b)
498{
499 const struct memprof_stats *l = (const struct memprof_stats *)a;
500 const struct memprof_stats *r = (const struct memprof_stats *)b;
501
502 if (l->alloc_tot + l->free_tot > r->alloc_tot + r->free_tot)
503 return -1;
504 else if (l->alloc_tot + l->free_tot < r->alloc_tot + r->free_tot)
505 return 1;
506 else
507 return 0;
508}
Willy Tarreauf1c8a382021-05-13 10:00:17 +0200509
510static int cmp_memprof_addr(const void *a, const void *b)
511{
512 const struct memprof_stats *l = (const struct memprof_stats *)a;
513 const struct memprof_stats *r = (const struct memprof_stats *)b;
514
515 if (l->caller > r->caller)
516 return -1;
517 else if (l->caller < r->caller)
518 return 1;
519 else
520 return 0;
521}
Willy Tarreau993d44d2021-05-05 18:07:02 +0200522#endif // USE_MEMORY_PROFILING
523
Willy Tarreau75c62c22018-11-22 11:02:09 +0100524/* This function dumps all profiling settings. It returns 0 if the output
525 * buffer is full and it needs to be called again, otherwise non-zero.
Willy Tarreau637d85a2021-05-05 17:33:27 +0200526 * It dumps some parts depending on the following states:
527 * ctx.cli.i0:
528 * 0, 4: dump status, then jump to 1 if 0
529 * 1, 5: dump tasks, then jump to 2 if 1
530 * 2, 6: dump memory, then stop
531 * ctx.cli.i1:
532 * restart line for each step (starts at zero)
533 * ctx.cli.o0:
534 * may contain a configured max line count for each step (0=not set)
Willy Tarreauf1c8a382021-05-13 10:00:17 +0200535 * ctx.cli.o1:
536 * 0: sort by usage
537 * 1: sort by address
Willy Tarreau75c62c22018-11-22 11:02:09 +0100538 */
539static int cli_io_handler_show_profiling(struct appctx *appctx)
540{
Willy Tarreau1bd67e92021-01-29 00:07:40 +0100541 struct sched_activity tmp_activity[256] __attribute__((aligned(64)));
Willy Tarreau3e109ee2021-08-28 12:04:25 +0200542#ifdef USE_MEMORY_PROFILING
Willy Tarreau993d44d2021-05-05 18:07:02 +0200543 struct memprof_stats tmp_memstats[MEMPROF_HASH_BUCKETS + 1];
Willy Tarreauf5fb8582021-05-11 14:06:24 +0200544 unsigned long long tot_alloc_calls, tot_free_calls;
545 unsigned long long tot_alloc_bytes, tot_free_bytes;
Willy Tarreau993d44d2021-05-05 18:07:02 +0200546#endif
Willy Tarreau75c62c22018-11-22 11:02:09 +0100547 struct stream_interface *si = appctx->owner;
Willy Tarreau1bd67e92021-01-29 00:07:40 +0100548 struct buffer *name_buffer = get_trash_chunk();
Willy Tarreaud2d33482019-04-25 17:09:07 +0200549 const char *str;
Willy Tarreau637d85a2021-05-05 17:33:27 +0200550 int max_lines;
Willy Tarreau1bd67e92021-01-29 00:07:40 +0100551 int i, max;
Willy Tarreau75c62c22018-11-22 11:02:09 +0100552
553 if (unlikely(si_ic(si)->flags & (CF_WRITE_ERROR|CF_SHUTW)))
554 return 1;
555
556 chunk_reset(&trash);
557
Willy Tarreaud2d33482019-04-25 17:09:07 +0200558 switch (profiling & HA_PROF_TASKS_MASK) {
Willy Tarreauaa622b82021-01-28 21:44:22 +0100559 case HA_PROF_TASKS_AOFF: str="auto-off"; break;
560 case HA_PROF_TASKS_AON: str="auto-on"; break;
Willy Tarreaud2d33482019-04-25 17:09:07 +0200561 case HA_PROF_TASKS_ON: str="on"; break;
562 default: str="off"; break;
563 }
564
Willy Tarreau637d85a2021-05-05 17:33:27 +0200565 if ((appctx->ctx.cli.i0 & 3) != 0)
566 goto skip_status;
Willy Tarreau1bd67e92021-01-29 00:07:40 +0100567
Willy Tarreaud2d33482019-04-25 17:09:07 +0200568 chunk_printf(&trash,
Willy Tarreau00dd44f2021-05-05 16:44:23 +0200569 "Per-task CPU profiling : %-8s # set profiling tasks {on|auto|off}\n"
570 "Memory usage profiling : %-8s # set profiling memory {on|off}\n",
571 str, (profiling & HA_PROF_MEMORY) ? "on" : "off");
Willy Tarreau75c62c22018-11-22 11:02:09 +0100572
Willy Tarreau637d85a2021-05-05 17:33:27 +0200573 if (ci_putchk(si_ic(si), &trash) == -1) {
574 /* failed, try again */
575 si_rx_room_blk(si);
576 return 0;
577 }
578
579 appctx->ctx.cli.i1 = 0; // reset first line to dump
580 if ((appctx->ctx.cli.i0 & 4) == 0)
581 appctx->ctx.cli.i0++; // next step
582
583 skip_status:
584 if ((appctx->ctx.cli.i0 & 3) != 1)
585 goto skip_tasks;
Willy Tarreau1bd67e92021-01-29 00:07:40 +0100586
Willy Tarreau637d85a2021-05-05 17:33:27 +0200587 memcpy(tmp_activity, sched_activity, sizeof(tmp_activity));
Willy Tarreauf1c8a382021-05-13 10:00:17 +0200588 if (appctx->ctx.cli.o1)
589 qsort(tmp_activity, 256, sizeof(tmp_activity[0]), cmp_sched_activity_addr);
590 else
591 qsort(tmp_activity, 256, sizeof(tmp_activity[0]), cmp_sched_activity_calls);
Willy Tarreau637d85a2021-05-05 17:33:27 +0200592
593 if (!appctx->ctx.cli.i1)
594 chunk_appendf(&trash, "Tasks activity:\n"
595 " function calls cpu_tot cpu_avg lat_tot lat_avg\n");
596
597 max_lines = appctx->ctx.cli.o0;
598 if (!max_lines)
599 max_lines = 256;
600
601 for (i = appctx->ctx.cli.i1; i < max_lines && tmp_activity[i].calls; i++) {
602 appctx->ctx.cli.i1 = i;
Willy Tarreau1bd67e92021-01-29 00:07:40 +0100603 chunk_reset(name_buffer);
604
605 if (!tmp_activity[i].func)
606 chunk_printf(name_buffer, "other");
607 else
608 resolve_sym_name(name_buffer, "", tmp_activity[i].func);
609
610 /* reserve 35 chars for name+' '+#calls, knowing that longer names
611 * are often used for less often called functions.
612 */
613 max = 35 - name_buffer->data;
614 if (max < 1)
615 max = 1;
616 chunk_appendf(&trash, " %s%*llu", name_buffer->area, max, (unsigned long long)tmp_activity[i].calls);
617
618 print_time_short(&trash, " ", tmp_activity[i].cpu_time, "");
619 print_time_short(&trash, " ", tmp_activity[i].cpu_time / tmp_activity[i].calls, "");
620 print_time_short(&trash, " ", tmp_activity[i].lat_time, "");
621 print_time_short(&trash, " ", tmp_activity[i].lat_time / tmp_activity[i].calls, "\n");
Willy Tarreau637d85a2021-05-05 17:33:27 +0200622
623 if (ci_putchk(si_ic(si), &trash) == -1) {
624 /* failed, try again */
625 si_rx_room_blk(si);
626 return 0;
627 }
Willy Tarreau1bd67e92021-01-29 00:07:40 +0100628 }
629
Willy Tarreau75c62c22018-11-22 11:02:09 +0100630 if (ci_putchk(si_ic(si), &trash) == -1) {
631 /* failed, try again */
632 si_rx_room_blk(si);
633 return 0;
634 }
Willy Tarreau637d85a2021-05-05 17:33:27 +0200635
636 appctx->ctx.cli.i1 = 0; // reset first line to dump
637 if ((appctx->ctx.cli.i0 & 4) == 0)
638 appctx->ctx.cli.i0++; // next step
639
640 skip_tasks:
641
Willy Tarreau3e109ee2021-08-28 12:04:25 +0200642#ifdef USE_MEMORY_PROFILING
Willy Tarreau993d44d2021-05-05 18:07:02 +0200643 if ((appctx->ctx.cli.i0 & 3) != 2)
644 goto skip_mem;
645
646 memcpy(tmp_memstats, memprof_stats, sizeof(tmp_memstats));
Willy Tarreauf1c8a382021-05-13 10:00:17 +0200647 if (appctx->ctx.cli.o1)
648 qsort(tmp_memstats, MEMPROF_HASH_BUCKETS+1, sizeof(tmp_memstats[0]), cmp_memprof_addr);
649 else
650 qsort(tmp_memstats, MEMPROF_HASH_BUCKETS+1, sizeof(tmp_memstats[0]), cmp_memprof_stats);
Willy Tarreau993d44d2021-05-05 18:07:02 +0200651
652 if (!appctx->ctx.cli.i1)
653 chunk_appendf(&trash,
654 "Alloc/Free statistics by call place:\n"
Willy Tarreau616491b2021-05-11 09:26:23 +0200655 " Calls | Tot Bytes | Caller and method\n"
Willy Tarreau993d44d2021-05-05 18:07:02 +0200656 "<- alloc -> <- free ->|<-- alloc ---> <-- free ---->|\n");
657
658 max_lines = appctx->ctx.cli.o0;
659 if (!max_lines)
660 max_lines = MEMPROF_HASH_BUCKETS + 1;
661
662 for (i = appctx->ctx.cli.i1; i < max_lines; i++) {
663 struct memprof_stats *entry = &tmp_memstats[i];
664
665 appctx->ctx.cli.i1 = i;
666 if (!entry->alloc_calls && !entry->free_calls)
667 continue;
668 chunk_appendf(&trash, "%11llu %11llu %14llu %14llu| %16p ",
669 entry->alloc_calls, entry->free_calls,
670 entry->alloc_tot, entry->free_tot,
671 entry->caller);
672
673 if (entry->caller)
674 resolve_sym_name(&trash, NULL, entry->caller);
675 else
676 chunk_appendf(&trash, "[other]");
677
Willy Tarreau60786932021-10-22 16:26:12 +0200678 chunk_appendf(&trash," %s(%lld)", memprof_methods[entry->method],
Willy Tarreau616491b2021-05-11 09:26:23 +0200679 (long long)(entry->alloc_tot - entry->free_tot) / (long long)(entry->alloc_calls + entry->free_calls));
Willy Tarreau993d44d2021-05-05 18:07:02 +0200680
Willy Tarreau60786932021-10-22 16:26:12 +0200681 if (entry->alloc_tot && entry->free_tot) {
682 /* that's a realloc, show the total diff to help spot leaks */
683 chunk_appendf(&trash," [delta=%lld]", (long long)(entry->alloc_tot - entry->free_tot));
684 }
685
686 chunk_appendf(&trash, "\n");
687
Willy Tarreau993d44d2021-05-05 18:07:02 +0200688 if (ci_putchk(si_ic(si), &trash) == -1) {
689 si_rx_room_blk(si);
690 return 0;
691 }
692 }
693
694 if (ci_putchk(si_ic(si), &trash) == -1) {
695 si_rx_room_blk(si);
696 return 0;
697 }
698
Willy Tarreauf5fb8582021-05-11 14:06:24 +0200699 tot_alloc_calls = tot_free_calls = tot_alloc_bytes = tot_free_bytes = 0;
700 for (i = 0; i < max_lines; i++) {
701 tot_alloc_calls += tmp_memstats[i].alloc_calls;
702 tot_free_calls += tmp_memstats[i].free_calls;
703 tot_alloc_bytes += tmp_memstats[i].alloc_tot;
704 tot_free_bytes += tmp_memstats[i].free_tot;
705 }
706
707 chunk_appendf(&trash,
708 "-----------------------|-----------------------------|\n"
709 "%11llu %11llu %14llu %14llu| <- Total; Delta_calls=%lld; Delta_bytes=%lld\n",
710 tot_alloc_calls, tot_free_calls,
711 tot_alloc_bytes, tot_free_bytes,
712 tot_alloc_calls - tot_free_calls,
713 tot_alloc_bytes - tot_free_bytes);
714
715 if (ci_putchk(si_ic(si), &trash) == -1) {
716 si_rx_room_blk(si);
717 return 0;
718 }
719
Willy Tarreau993d44d2021-05-05 18:07:02 +0200720 appctx->ctx.cli.i1 = 0; // reset first line to dump
721 if ((appctx->ctx.cli.i0 & 4) == 0)
722 appctx->ctx.cli.i0++; // next step
723
724 skip_mem:
725#endif // USE_MEMORY_PROFILING
726
Willy Tarreau75c62c22018-11-22 11:02:09 +0100727 return 1;
728}
729
Willy Tarreauf1c8a382021-05-13 10:00:17 +0200730/* parse a "show profiling" command. It returns 1 on failure, 0 if it starts to dump.
731 * - cli.i0 is set to the first state (0=all, 4=status, 5=tasks, 6=memory)
732 * - cli.o1 is set to 1 if the output must be sorted by addr instead of usage
733 * - cli.o0 is set to the number of lines of output
734 */
Willy Tarreau42712cb2021-05-05 17:48:13 +0200735static int cli_parse_show_profiling(char **args, char *payload, struct appctx *appctx, void *private)
736{
Willy Tarreauf1c8a382021-05-13 10:00:17 +0200737 int arg;
738
Willy Tarreau42712cb2021-05-05 17:48:13 +0200739 if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))
740 return 1;
741
Willy Tarreauf1c8a382021-05-13 10:00:17 +0200742 for (arg = 2; *args[arg]; arg++) {
743 if (strcmp(args[arg], "all") == 0) {
744 appctx->ctx.cli.i0 = 0; // will cycle through 0,1,2; default
745 }
746 else if (strcmp(args[arg], "status") == 0) {
747 appctx->ctx.cli.i0 = 4; // will visit status only
748 }
749 else if (strcmp(args[arg], "tasks") == 0) {
750 appctx->ctx.cli.i0 = 5; // will visit tasks only
751 }
752 else if (strcmp(args[arg], "memory") == 0) {
753 appctx->ctx.cli.i0 = 6; // will visit memory only
754 }
755 else if (strcmp(args[arg], "byaddr") == 0) {
756 appctx->ctx.cli.o1 = 1; // sort output by address instead of usage
757 }
758 else if (isdigit((unsigned char)*args[arg])) {
759 appctx->ctx.cli.o0 = atoi(args[arg]); // number of entries to dump
760 }
761 else
762 return cli_err(appctx, "Expects either 'all', 'status', 'tasks', 'memory', 'byaddr' or a max number of output lines.\n");
Willy Tarreau42712cb2021-05-05 17:48:13 +0200763 }
764 return 0;
765}
766
Willy Tarreau7eff06e2021-01-29 11:32:55 +0100767/* This function scans all threads' run queues and collects statistics about
768 * running tasks. It returns 0 if the output buffer is full and it needs to be
769 * called again, otherwise non-zero.
770 */
771static int cli_io_handler_show_tasks(struct appctx *appctx)
772{
773 struct sched_activity tmp_activity[256] __attribute__((aligned(64)));
774 struct stream_interface *si = appctx->owner;
775 struct buffer *name_buffer = get_trash_chunk();
776 struct sched_activity *entry;
777 const struct tasklet *tl;
778 const struct task *t;
779 uint64_t now_ns, lat;
780 struct eb32sc_node *rqnode;
781 uint64_t tot_calls;
782 int thr, queue;
783 int i, max;
784
785 if (unlikely(si_ic(si)->flags & (CF_WRITE_ERROR|CF_SHUTW)))
786 return 1;
787
788 /* It's not possible to scan queues in small chunks and yield in the
789 * middle of the dump and come back again. So what we're doing instead
790 * is to freeze all threads and inspect their queues at once as fast as
791 * possible, using a sched_activity array to collect metrics with
792 * limited collision, then we'll report statistics only. The tasks'
793 * #calls will reflect the number of occurrences, and the lat_time will
Willy Tarreau75f72332021-01-29 15:04:16 +0100794 * reflect the latency when set. We prefer to take the time before
795 * calling thread_isolate() so that the wait time doesn't impact the
796 * measurement accuracy. However this requires to take care of negative
797 * times since tasks might be queued after we retrieve it.
Willy Tarreau7eff06e2021-01-29 11:32:55 +0100798 */
799
800 now_ns = now_mono_time();
801 memset(tmp_activity, 0, sizeof(tmp_activity));
802
803 thread_isolate();
804
805 /* 1. global run queue */
806
807#ifdef USE_THREAD
808 rqnode = eb32sc_first(&rqueue, ~0UL);
809 while (rqnode) {
810 t = eb32sc_entry(rqnode, struct task, rq);
811 entry = sched_activity_entry(tmp_activity, t->process);
812 if (t->call_date) {
813 lat = now_ns - t->call_date;
Willy Tarreau75f72332021-01-29 15:04:16 +0100814 if ((int64_t)lat > 0)
815 entry->lat_time += lat;
Willy Tarreau7eff06e2021-01-29 11:32:55 +0100816 }
817 entry->calls++;
818 rqnode = eb32sc_next(rqnode, ~0UL);
819 }
820#endif
821 /* 2. all threads's local run queues */
822 for (thr = 0; thr < global.nbthread; thr++) {
823 /* task run queue */
824 rqnode = eb32sc_first(&task_per_thread[thr].rqueue, ~0UL);
825 while (rqnode) {
826 t = eb32sc_entry(rqnode, struct task, rq);
827 entry = sched_activity_entry(tmp_activity, t->process);
828 if (t->call_date) {
829 lat = now_ns - t->call_date;
Willy Tarreau75f72332021-01-29 15:04:16 +0100830 if ((int64_t)lat > 0)
831 entry->lat_time += lat;
Willy Tarreau7eff06e2021-01-29 11:32:55 +0100832 }
833 entry->calls++;
834 rqnode = eb32sc_next(rqnode, ~0UL);
835 }
836
837 /* shared tasklet list */
838 list_for_each_entry(tl, mt_list_to_list(&task_per_thread[thr].shared_tasklet_list), list) {
839 t = (const struct task *)tl;
840 entry = sched_activity_entry(tmp_activity, t->process);
841 if (!TASK_IS_TASKLET(t) && t->call_date) {
842 lat = now_ns - t->call_date;
Willy Tarreau75f72332021-01-29 15:04:16 +0100843 if ((int64_t)lat > 0)
844 entry->lat_time += lat;
Willy Tarreau7eff06e2021-01-29 11:32:55 +0100845 }
846 entry->calls++;
847 }
848
849 /* classful tasklets */
850 for (queue = 0; queue < TL_CLASSES; queue++) {
851 list_for_each_entry(tl, &task_per_thread[thr].tasklets[queue], list) {
852 t = (const struct task *)tl;
853 entry = sched_activity_entry(tmp_activity, t->process);
854 if (!TASK_IS_TASKLET(t) && t->call_date) {
855 lat = now_ns - t->call_date;
Willy Tarreau75f72332021-01-29 15:04:16 +0100856 if ((int64_t)lat > 0)
857 entry->lat_time += lat;
Willy Tarreau7eff06e2021-01-29 11:32:55 +0100858 }
859 entry->calls++;
860 }
861 }
862 }
863
864 /* hopefully we're done */
865 thread_release();
866
867 chunk_reset(&trash);
868
869 tot_calls = 0;
870 for (i = 0; i < 256; i++)
871 tot_calls += tmp_activity[i].calls;
872
Willy Tarreauf1c8a382021-05-13 10:00:17 +0200873 qsort(tmp_activity, 256, sizeof(tmp_activity[0]), cmp_sched_activity_calls);
Willy Tarreau7eff06e2021-01-29 11:32:55 +0100874
875 chunk_appendf(&trash, "Running tasks: %d (%d threads)\n"
876 " function places %% lat_tot lat_avg\n",
877 (int)tot_calls, global.nbthread);
878
879 for (i = 0; i < 256 && tmp_activity[i].calls; i++) {
880 chunk_reset(name_buffer);
881
882 if (!tmp_activity[i].func)
883 chunk_printf(name_buffer, "other");
884 else
885 resolve_sym_name(name_buffer, "", tmp_activity[i].func);
886
887 /* reserve 35 chars for name+' '+#calls, knowing that longer names
888 * are often used for less often called functions.
889 */
890 max = 35 - name_buffer->data;
891 if (max < 1)
892 max = 1;
893 chunk_appendf(&trash, " %s%*llu %3d.%1d",
894 name_buffer->area, max, (unsigned long long)tmp_activity[i].calls,
895 (int)(100ULL * tmp_activity[i].calls / tot_calls),
896 (int)((1000ULL * tmp_activity[i].calls / tot_calls)%10));
897 print_time_short(&trash, " ", tmp_activity[i].lat_time, "");
898 print_time_short(&trash, " ", tmp_activity[i].lat_time / tmp_activity[i].calls, "\n");
899 }
900
901 if (ci_putchk(si_ic(si), &trash) == -1) {
902 /* failed, try again */
903 si_rx_room_blk(si);
904 return 0;
905 }
906 return 1;
907}
908
Willy Tarreau75c62c22018-11-22 11:02:09 +0100909/* config keyword parsers */
910static struct cfg_kw_list cfg_kws = {ILH, {
Willy Tarreauca3afc22021-05-05 18:33:19 +0200911#ifdef USE_MEMORY_PROFILING
912 { CFG_GLOBAL, "profiling.memory", cfg_parse_prof_memory },
913#endif
Willy Tarreau75c62c22018-11-22 11:02:09 +0100914 { CFG_GLOBAL, "profiling.tasks", cfg_parse_prof_tasks },
915 { 0, NULL, NULL }
916}};
917
Willy Tarreau0108d902018-11-25 19:14:37 +0100918INITCALL1(STG_REGISTER, cfg_register_keywords, &cfg_kws);
919
Willy Tarreau75c62c22018-11-22 11:02:09 +0100920/* register cli keywords */
921static struct cli_kw_list cli_kws = {{ },{
Daniel Corbett67b3cef2021-05-10 14:08:40 -0400922 { { "set", "profiling", NULL }, "set profiling <what> {auto|on|off} : enable/disable resource profiling (tasks,memory)", cli_parse_set_profiling, NULL },
Willy Tarreauf1c8a382021-05-13 10:00:17 +0200923 { { "show", "profiling", NULL }, "show profiling [<what>|<#lines>|byaddr]*: show profiling state (all,status,tasks,memory)", cli_parse_show_profiling, cli_io_handler_show_profiling, NULL },
Willy Tarreaub205bfd2021-05-07 11:38:37 +0200924 { { "show", "tasks", NULL }, "show tasks : show running tasks", NULL, cli_io_handler_show_tasks, NULL },
Willy Tarreau75c62c22018-11-22 11:02:09 +0100925 {{},}
926}};
927
Willy Tarreau0108d902018-11-25 19:14:37 +0100928INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);