blob: 5d256209b5d953bd03b6fc7efd78c53248c4f558 [file] [log] [blame]
Willy Tarreau72c28532009-01-22 18:56:50 +01001/*
Willy Tarreaud8fc1102010-09-12 17:56:16 +02002 * haproxy log statistics reporter
Willy Tarreau72c28532009-01-22 18:56:50 +01003 *
Willy Tarreau8a09b662012-10-10 10:26:22 +02004 * Copyright 2000-2012 Willy Tarreau <w@1wt.eu>
Willy Tarreau72c28532009-01-22 18:56:50 +01005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
Willy Tarreau72c28532009-01-22 18:56:50 +010013#include <errno.h>
14#include <fcntl.h>
15#include <stdio.h>
16#include <stdlib.h>
17#include <syslog.h>
18#include <string.h>
19#include <unistd.h>
20#include <ctype.h>
Olivier Burgarde97b9042014-05-22 16:44:59 +020021#include <time.h>
Willy Tarreau72c28532009-01-22 18:56:50 +010022
Willy Tarreau8d2b7772020-05-27 10:58:19 +020023#include <import/eb32tree.h>
24#include <import/eb64tree.h>
25#include <import/ebistree.h>
26#include <import/ebsttree.h>
Willy Tarreau72c28532009-01-22 18:56:50 +010027
Willy Tarreaud2201062010-05-27 18:17:30 +020028#define SOURCE_FIELD 5
Willy Tarreau72c28532009-01-22 18:56:50 +010029#define ACCEPT_FIELD 6
Willy Tarreaud2201062010-05-27 18:17:30 +020030#define SERVER_FIELD 8
Willy Tarreau72c28532009-01-22 18:56:50 +010031#define TIME_FIELD 9
32#define STATUS_FIELD 10
Baptiste61aaad02012-09-08 23:10:03 +020033#define BYTES_SENT_FIELD 11
Willy Tarreaud8fc1102010-09-12 17:56:16 +020034#define TERM_CODES_FIELD 14
Willy Tarreau72c28532009-01-22 18:56:50 +010035#define CONN_FIELD 15
Willy Tarreau08911ff2011-10-13 13:28:36 +020036#define QUEUE_LEN_FIELD 16
Willy Tarreauabe45b62010-10-28 20:33:46 +020037#define METH_FIELD 17
38#define URL_FIELD 18
Willy Tarreau72c28532009-01-22 18:56:50 +010039#define MAXLINE 16384
40#define QBITS 4
41
Willy Tarreaudf6f0d12011-07-10 18:15:08 +020042#define SEP(c) ((unsigned char)(c) <= ' ')
43#define SKIP_CHAR(p,c) do { while (1) { int __c = (unsigned char)*p++; if (__c == c) break; if (__c <= ' ') { p--; break; } } } while (0)
Willy Tarreau72c28532009-01-22 18:56:50 +010044
45/* [0] = err/date, [1] = req, [2] = conn, [3] = resp, [4] = data */
46static struct eb_root timers[5] = {
47 EB_ROOT_UNIQUE, EB_ROOT_UNIQUE, EB_ROOT_UNIQUE,
48 EB_ROOT_UNIQUE, EB_ROOT_UNIQUE,
49};
50
51struct timer {
52 struct eb32_node node;
53 unsigned int count;
54};
55
Willy Tarreaud2201062010-05-27 18:17:30 +020056struct srv_st {
57 unsigned int st_cnt[6]; /* 0xx to 5xx */
58 unsigned int nb_ct, nb_rt, nb_ok;
59 unsigned long long cum_ct, cum_rt;
60 struct ebmb_node node;
61 /* don't put anything else here, the server name will be there */
62};
Willy Tarreau72c28532009-01-22 18:56:50 +010063
Willy Tarreauabe45b62010-10-28 20:33:46 +020064struct url_stat {
65 union {
66 struct ebpt_node url;
67 struct eb64_node val;
68 } node;
69 char *url;
70 unsigned long long total_time; /* sum(all reqs' times) */
71 unsigned long long total_time_ok; /* sum(all OK reqs' times) */
Baptiste61aaad02012-09-08 23:10:03 +020072 unsigned long long total_bytes_sent; /* sum(all bytes sent) */
Willy Tarreauabe45b62010-10-28 20:33:46 +020073 unsigned int nb_err, nb_req;
74};
75
Willy Tarreau72c28532009-01-22 18:56:50 +010076#define FILT_COUNT_ONLY 0x01
77#define FILT_INVERT 0x02
78#define FILT_QUIET 0x04
79#define FILT_ERRORS_ONLY 0x08
80#define FILT_ACC_DELAY 0x10
81#define FILT_ACC_COUNT 0x20
82#define FILT_GRAPH_TIMERS 0x40
Willy Tarreau214c2032009-02-20 11:02:32 +010083#define FILT_PERCENTILE 0x80
Willy Tarreau5bdfd962009-10-14 15:16:29 +020084#define FILT_TIME_RESP 0x100
85
86#define FILT_INVERT_ERRORS 0x200
87#define FILT_INVERT_TIME_RESP 0x400
Willy Tarreau72c28532009-01-22 18:56:50 +010088
Willy Tarreau0f423a72010-05-03 10:50:54 +020089#define FILT_COUNT_STATUS 0x800
Willy Tarreaud2201062010-05-27 18:17:30 +020090#define FILT_COUNT_SRV_STATUS 0x1000
Willy Tarreaud8fc1102010-09-12 17:56:16 +020091#define FILT_COUNT_TERM_CODES 0x2000
Willy Tarreau0f423a72010-05-03 10:50:54 +020092
Willy Tarreauabe45b62010-10-28 20:33:46 +020093#define FILT_COUNT_URL_ONLY 0x004000
94#define FILT_COUNT_URL_COUNT 0x008000
95#define FILT_COUNT_URL_ERR 0x010000
96#define FILT_COUNT_URL_TTOT 0x020000
97#define FILT_COUNT_URL_TAVG 0x040000
98#define FILT_COUNT_URL_TTOTO 0x080000
99#define FILT_COUNT_URL_TAVGO 0x100000
Willy Tarreauabe45b62010-10-28 20:33:46 +0200100
Willy Tarreau70c428f2011-07-10 17:27:40 +0200101#define FILT_HTTP_ONLY 0x200000
Willy Tarreaud3007ff2011-09-05 02:07:23 +0200102#define FILT_TERM_CODE_NAME 0x400000
Hervé COMMOWICK927cddd2011-08-10 17:42:41 +0200103#define FILT_INVERT_TERM_CODE_NAME 0x800000
Willy Tarreau70c428f2011-07-10 17:27:40 +0200104
Willy Tarreaud3007ff2011-09-05 02:07:23 +0200105#define FILT_HTTP_STATUS 0x1000000
106#define FILT_INVERT_HTTP_STATUS 0x2000000
Willy Tarreau08911ff2011-10-13 13:28:36 +0200107#define FILT_QUEUE_ONLY 0x4000000
108#define FILT_QUEUE_SRV_ONLY 0x8000000
Willy Tarreaud3007ff2011-09-05 02:07:23 +0200109
Baptiste61aaad02012-09-08 23:10:03 +0200110#define FILT_COUNT_URL_BAVG 0x10000000
111#define FILT_COUNT_URL_BTOT 0x20000000
112
113#define FILT_COUNT_URL_ANY (FILT_COUNT_URL_ONLY|FILT_COUNT_URL_COUNT|FILT_COUNT_URL_ERR| \
114 FILT_COUNT_URL_TTOT|FILT_COUNT_URL_TAVG|FILT_COUNT_URL_TTOTO|FILT_COUNT_URL_TAVGO| \
115 FILT_COUNT_URL_BAVG|FILT_COUNT_URL_BTOT)
116
Willy Tarreau8a09b662012-10-10 10:26:22 +0200117#define FILT_COUNT_COOK_CODES 0x40000000
Willy Tarreau7cf479c2013-02-16 23:49:04 +0100118#define FILT_COUNT_IP_COUNT 0x80000000
Willy Tarreau8a09b662012-10-10 10:26:22 +0200119
Olivier Burgarde97b9042014-05-22 16:44:59 +0200120#define FILT2_TIMESTAMP 0x01
121
Willy Tarreau72c28532009-01-22 18:56:50 +0100122unsigned int filter = 0;
Olivier Burgarde97b9042014-05-22 16:44:59 +0200123unsigned int filter2 = 0;
Willy Tarreau72c28532009-01-22 18:56:50 +0100124unsigned int filter_invert = 0;
Willy Tarreau214c2032009-02-20 11:02:32 +0100125const char *line;
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200126int linenum = 0;
127int parse_err = 0;
128int lines_out = 0;
Willy Tarreau667c9052012-10-10 16:49:28 +0200129int lines_max = -1;
Willy Tarreau72c28532009-01-22 18:56:50 +0100130
Willy Tarreau214c2032009-02-20 11:02:32 +0100131const char *fgets2(FILE *stream);
Willy Tarreau72c28532009-01-22 18:56:50 +0100132
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200133void filter_count_url(const char *accept_field, const char *time_field, struct timer **tptr);
Willy Tarreau7cf479c2013-02-16 23:49:04 +0100134void filter_count_ip(const char *source_field, const char *accept_field, const char *time_field, struct timer **tptr);
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200135void filter_count_srv_status(const char *accept_field, const char *time_field, struct timer **tptr);
Willy Tarreau8a09b662012-10-10 10:26:22 +0200136void filter_count_cook_codes(const char *accept_field, const char *time_field, struct timer **tptr);
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200137void filter_count_term_codes(const char *accept_field, const char *time_field, struct timer **tptr);
138void filter_count_status(const char *accept_field, const char *time_field, struct timer **tptr);
139void filter_graphs(const char *accept_field, const char *time_field, struct timer **tptr);
140void filter_output_line(const char *accept_field, const char *time_field, struct timer **tptr);
141void filter_accept_holes(const char *accept_field, const char *time_field, struct timer **tptr);
142
Willy Tarreau615674c2012-01-23 08:15:51 +0100143void usage(FILE *output, const char *msg)
Willy Tarreau72c28532009-01-22 18:56:50 +0100144{
Willy Tarreau615674c2012-01-23 08:15:51 +0100145 fprintf(output,
Willy Tarreau72c28532009-01-22 18:56:50 +0100146 "%s"
Willy Tarreau615674c2012-01-23 08:15:51 +0100147 "Usage: halog [-h|--help] for long help\n"
Willy Tarreau667c9052012-10-10 16:49:28 +0200148 " halog [-q] [-c] [-m <lines>]\n"
Willy Tarreau7cf479c2013-02-16 23:49:04 +0100149 " {-cc|-gt|-pct|-st|-tc|-srv|-u|-uc|-ue|-ua|-ut|-uao|-uto|-uba|-ubt|-ic}\n"
Hervé COMMOWICK927cddd2011-08-10 17:42:41 +0200150 " [-s <skip>] [-e|-E] [-H] [-rt|-RT <time>] [-ad <delay>] [-ac <count>]\n"
Olivier Burgarde97b9042014-05-22 16:44:59 +0200151 " [-v] [-Q|-QS] [-tcn|-TCN <termcode>] [ -hs|-HS [min][:[max]] ] [ -time [min][:[max]] ] < log\n"
Willy Tarreau72c28532009-01-22 18:56:50 +0100152 "\n",
153 msg ? msg : ""
154 );
Willy Tarreau615674c2012-01-23 08:15:51 +0100155}
156
157void die(const char *msg)
158{
159 usage(stderr, msg);
Willy Tarreau72c28532009-01-22 18:56:50 +0100160 exit(1);
161}
162
Willy Tarreau615674c2012-01-23 08:15:51 +0100163void help()
164{
165 usage(stdout, NULL);
166 printf(
167 "Input filters (several filters may be combined) :\n"
168 " -H only match lines containing HTTP logs (ignore TCP)\n"
169 " -E only match lines without any error (no 5xx status)\n"
170 " -e only match lines with errors (status 5xx or negative)\n"
171 " -rt|-RT <time> only match response times larger|smaller than <time>\n"
172 " -Q|-QS only match queued requests (any queue|server queue)\n"
173 " -tcn|-TCN <code> only match requests with/without termination code <code>\n"
174 " -hs|-HS <[min][:][max]> only match requests with HTTP status codes within/not\n"
175 " within min..max. Any of them may be omitted. Exact\n"
176 " code is checked for if no ':' is specified.\n"
Olivier Burgarde97b9042014-05-22 16:44:59 +0200177 " -time <[min][:max]> only match requests recorded between timestamps.\n"
178 " Any of them may be omitted.\n"
Willy Tarreau615674c2012-01-23 08:15:51 +0100179 "Modifiers\n"
180 " -v invert the input filtering condition\n"
181 " -q don't report errors/warnings\n"
Willy Tarreau667c9052012-10-10 16:49:28 +0200182 " -m <lines> limit output to the first <lines> lines\n"
Aleksandar Lazicf2b5d752017-12-05 01:35:21 +0100183 " -s <skip_n_fields> skip n fields from the beginning of a line (default %d)\n"
184 " you can also use -n to start from earlier then field %d\n"
185 "\n"
Willy Tarreau615674c2012-01-23 08:15:51 +0100186 "Output filters - only one may be used at a time\n"
187 " -c only report the number of lines that would have been printed\n"
188 " -pct output connect and response times percentiles\n"
189 " -st output number of requests per HTTP status code\n"
Willy Tarreau8a09b662012-10-10 10:26:22 +0200190 " -cc output number of requests per cookie code (2 chars)\n"
Willy Tarreau615674c2012-01-23 08:15:51 +0100191 " -tc output number of requests per termination code (2 chars)\n"
192 " -srv output statistics per server (time, requests, errors)\n"
Aleksandar Lazi6112f5c2020-05-15 22:58:30 +0200193 " -ic output statistics per ip count (time, requests, errors)\n"
Willy Tarreau615674c2012-01-23 08:15:51 +0100194 " -u* output statistics per URL (time, requests, errors)\n"
195 " Additional characters indicate the output sorting key :\n"
196 " -u : by URL, -uc : request count, -ue : error count\n"
Willy Tarreau4201df72012-10-10 14:57:35 +0200197 " -ua : average response time, -ut : average total time\n"
Willy Tarreau615674c2012-01-23 08:15:51 +0100198 " -uao, -uto: average times computed on valid ('OK') requests\n"
Aleksandar Lazicf2b5d752017-12-05 01:35:21 +0100199 " -uba, -ubt: average bytes returned, total bytes returned\n",
200 SOURCE_FIELD,SOURCE_FIELD
Willy Tarreau615674c2012-01-23 08:15:51 +0100201 );
202 exit(0);
203}
204
Willy Tarreau72c28532009-01-22 18:56:50 +0100205
206/* return pointer to first char not part of current field starting at <p>. */
Willy Tarreauf9042062011-09-10 12:26:35 +0200207
208#if defined(__i386__)
209/* this one is always faster on 32-bits */
210static inline const char *field_stop(const char *p)
211{
212 asm(
213 /* Look for spaces */
214 "4: \n\t"
215 "inc %0 \n\t"
216 "cmpb $0x20, -1(%0) \n\t"
217 "ja 4b \n\t"
218 "jz 3f \n\t"
219
220 /* we only get there for control chars 0..31. Leave if we find '\0' */
221 "cmpb $0x0, -1(%0) \n\t"
222 "jnz 4b \n\t"
223
224 /* return %0-1 = position of the last char we checked */
225 "3: \n\t"
226 "dec %0 \n\t"
227 : "=r" (p)
228 : "0" (p)
229 );
230 return p;
231}
232#else
Willy Tarreau72c28532009-01-22 18:56:50 +0100233const char *field_stop(const char *p)
234{
235 unsigned char c;
236
237 while (1) {
238 c = *(p++);
239 if (c > ' ')
240 continue;
Willy Tarreau14389e72011-07-10 22:11:17 +0200241 if (c == ' ' || c == 0)
Willy Tarreau72c28532009-01-22 18:56:50 +0100242 break;
243 }
244 return p - 1;
245}
Willy Tarreauf9042062011-09-10 12:26:35 +0200246#endif
Willy Tarreau72c28532009-01-22 18:56:50 +0100247
248/* return field <field> (starting from 1) in string <p>. Only consider
249 * contiguous spaces (or tabs) as one delimiter. May return pointer to
250 * last char if field is not found. Equivalent to awk '{print $field}'.
251 */
252const char *field_start(const char *p, int field)
253{
Willy Tarreauf9042062011-09-10 12:26:35 +0200254#ifndef PREFER_ASM
Willy Tarreau72c28532009-01-22 18:56:50 +0100255 unsigned char c;
256 while (1) {
257 /* skip spaces */
258 while (1) {
Willy Tarreauf9042062011-09-10 12:26:35 +0200259 c = *(p++);
Willy Tarreau72c28532009-01-22 18:56:50 +0100260 if (c > ' ')
261 break;
Willy Tarreau14389e72011-07-10 22:11:17 +0200262 if (c == ' ')
Willy Tarreauf9042062011-09-10 12:26:35 +0200263 continue;
Willy Tarreau72c28532009-01-22 18:56:50 +0100264 if (!c) /* end of line */
Willy Tarreauf9042062011-09-10 12:26:35 +0200265 return p-1;
Willy Tarreau72c28532009-01-22 18:56:50 +0100266 /* other char => new field */
267 break;
Willy Tarreau72c28532009-01-22 18:56:50 +0100268 }
269
270 /* start of field */
271 field--;
272 if (!field)
Willy Tarreauf9042062011-09-10 12:26:35 +0200273 return p-1;
Willy Tarreau72c28532009-01-22 18:56:50 +0100274
275 /* skip this field */
276 while (1) {
277 c = *(p++);
Willy Tarreau14389e72011-07-10 22:11:17 +0200278 if (c == ' ')
Willy Tarreau72c28532009-01-22 18:56:50 +0100279 break;
Willy Tarreauf9042062011-09-10 12:26:35 +0200280 if (c > ' ')
281 continue;
Willy Tarreau72c28532009-01-22 18:56:50 +0100282 if (c == '\0')
Willy Tarreauf9042062011-09-10 12:26:35 +0200283 return p - 1;
Willy Tarreau72c28532009-01-22 18:56:50 +0100284 }
285 }
Willy Tarreauf9042062011-09-10 12:26:35 +0200286#else
287 /* This version works optimally on i386 and x86_64 but the code above
288 * shows similar performance. However, depending on the version of GCC
289 * used, inlining rules change and it may have difficulties to make
290 * efficient use of this code at other locations and could result in
291 * worse performance (eg: gcc 4.4). You may want to experience.
292 */
293 asm(
294 /* skip spaces */
295 "1: \n\t"
296 "inc %0 \n\t"
297 "cmpb $0x20, -1(%0) \n\t"
298 "ja 2f \n\t"
299 "jz 1b \n\t"
300
301 /* we only get there for control chars 0..31. Leave if we find '\0' */
302 "cmpb $0x0, -1(%0) \n\t"
303 "jz 3f \n\t"
304
305 /* start of field at [%0-1]. Check if we need to skip more fields */
306 "2: \n\t"
307 "dec %1 \n\t"
308 "jz 3f \n\t"
309
310 /* Look for spaces */
311 "4: \n\t"
312 "inc %0 \n\t"
313 "cmpb $0x20, -1(%0) \n\t"
314 "jz 1b \n\t"
315 "ja 4b \n\t"
316
317 /* we only get there for control chars 0..31. Leave if we find '\0' */
318 "cmpb $0x0, -1(%0) \n\t"
319 "jnz 4b \n\t"
320
321 /* return %0-1 = position of the last char we checked */
322 "3: \n\t"
323 "dec %0 \n\t"
324 : "=r" (p)
325 : "r" (field), "0" (p)
326 );
327 return p;
328#endif
Willy Tarreau72c28532009-01-22 18:56:50 +0100329}
330
331/* keep only the <bits> higher bits of <i> */
332static inline unsigned int quantify_u32(unsigned int i, int bits)
333{
334 int high;
335
336 if (!bits)
337 return 0;
338
339 if (i)
340 high = fls_auto(i); // 1 to 32
341 else
342 high = 0;
343
344 if (high <= bits)
345 return i;
346
347 return i & ~((1 << (high - bits)) - 1);
348}
349
350/* keep only the <bits> higher bits of the absolute value of <i>, as well as
351 * its sign. */
352static inline int quantify(int i, int bits)
353{
354 if (i >= 0)
355 return quantify_u32(i, bits);
356 else
357 return -quantify_u32(-i, bits);
358}
359
360/* Insert timer value <v> into tree <r>. A pre-allocated node must be passed
361 * in <alloc>. It may be NULL, in which case the function will allocate it
362 * itself. It will be reset to NULL once consumed. The caller is responsible
363 * for freeing the node once not used anymore. The node where the value was
364 * inserted is returned.
365 */
366struct timer *insert_timer(struct eb_root *r, struct timer **alloc, int v)
367{
368 struct timer *t = *alloc;
369 struct eb32_node *n;
370
371 if (!t) {
372 t = calloc(sizeof(*t), 1);
373 if (unlikely(!t)) {
374 fprintf(stderr, "%s: not enough memory\n", __FUNCTION__);
375 exit(1);
376 }
377 }
378 t->node.key = quantify(v, QBITS); // keep only the higher QBITS bits
379
380 n = eb32i_insert(r, &t->node);
381 if (n == &t->node)
382 t = NULL; /* node inserted, will malloc next time */
383
384 *alloc = t;
385 return container_of(n, struct timer, node);
386}
387
388/* Insert value value <v> into tree <r>. A pre-allocated node must be passed
389 * in <alloc>. It may be NULL, in which case the function will allocate it
390 * itself. It will be reset to NULL once consumed. The caller is responsible
391 * for freeing the node once not used anymore. The node where the value was
392 * inserted is returned.
393 */
394struct timer *insert_value(struct eb_root *r, struct timer **alloc, int v)
395{
396 struct timer *t = *alloc;
397 struct eb32_node *n;
398
399 if (!t) {
400 t = calloc(sizeof(*t), 1);
401 if (unlikely(!t)) {
402 fprintf(stderr, "%s: not enough memory\n", __FUNCTION__);
403 exit(1);
404 }
405 }
406 t->node.key = v;
407
408 n = eb32i_insert(r, &t->node);
409 if (n == &t->node)
410 t = NULL; /* node inserted, will malloc next time */
411
412 *alloc = t;
413 return container_of(n, struct timer, node);
414}
415
416int str2ic(const char *s)
417{
418 int i = 0;
419 int j, k;
420
421 if (*s != '-') {
422 /* positive number */
423 while (1) {
424 j = (*s++) - '0';
425 k = i * 10;
426 if ((unsigned)j > 9)
427 break;
428 i = k + j;
429 }
430 } else {
431 /* negative number */
432 s++;
433 while (1) {
434 j = (*s++) - '0';
435 k = i * 10;
436 if ((unsigned)j > 9)
437 break;
438 i = k - j;
439 }
440 }
441
442 return i;
443}
444
445
446/* Equivalent to strtoul with a length. */
447static inline unsigned int __strl2ui(const char *s, int len)
448{
449 unsigned int i = 0;
450 while (len-- > 0) {
451 i = i * 10 - '0';
452 i += (unsigned char)*s++;
453 }
454 return i;
455}
456
457unsigned int strl2ui(const char *s, int len)
458{
459 return __strl2ui(s, len);
460}
461
462/* Convert "[04/Dec/2008:09:49:40.555]" to an integer equivalent to the time of
463 * the day in milliseconds. It returns -1 for all unparsable values. The parser
464 * looks ugly but gcc emits far better code that way.
465 */
466int convert_date(const char *field)
467{
468 unsigned int h, m, s, ms;
469 unsigned char c;
Ryan O'Hara8cb99932017-12-15 10:21:39 -0600470 const char *e;
Willy Tarreau72c28532009-01-22 18:56:50 +0100471
472 h = m = s = ms = 0;
473 e = field;
474
475 /* skip the date */
476 while (1) {
477 c = *(e++);
478 if (c == ':')
479 break;
480 if (!c)
481 goto out_err;
482 }
483
484 /* hour + ':' */
Willy Tarreau72c28532009-01-22 18:56:50 +0100485 while (1) {
486 c = *(e++) - '0';
487 if (c > 9)
488 break;
489 h = h * 10 + c;
490 }
491 if (c == (unsigned char)(0 - '0'))
492 goto out_err;
493
494 /* minute + ':' */
Willy Tarreau72c28532009-01-22 18:56:50 +0100495 while (1) {
496 c = *(e++) - '0';
497 if (c > 9)
498 break;
499 m = m * 10 + c;
500 }
501 if (c == (unsigned char)(0 - '0'))
502 goto out_err;
503
504 /* second + '.' or ']' */
Willy Tarreau72c28532009-01-22 18:56:50 +0100505 while (1) {
506 c = *(e++) - '0';
507 if (c > 9)
508 break;
509 s = s * 10 + c;
510 }
511 if (c == (unsigned char)(0 - '0'))
512 goto out_err;
513
514 /* if there's a '.', we have milliseconds */
515 if (c == (unsigned char)('.' - '0')) {
516 /* millisecond second + ']' */
Willy Tarreau72c28532009-01-22 18:56:50 +0100517 while (1) {
518 c = *(e++) - '0';
519 if (c > 9)
520 break;
521 ms = ms * 10 + c;
522 }
523 if (c == (unsigned char)(0 - '0'))
524 goto out_err;
525 }
526 return (((h * 60) + m) * 60 + s) * 1000 + ms;
527 out_err:
528 return -1;
529}
530
Olivier Burgarde97b9042014-05-22 16:44:59 +0200531/* Convert "[04/Dec/2008:09:49:40.555]" to an unix timestamp.
532 * It returns -1 for all unparsable values. The parser
533 * looks ugly but gcc emits far better code that way.
534 */
535int convert_date_to_timestamp(const char *field)
536{
537 unsigned int d, mo, y, h, m, s;
538 unsigned char c;
Ryan O'Hara8cb99932017-12-15 10:21:39 -0600539 const char *e;
Olivier Burgarde97b9042014-05-22 16:44:59 +0200540 time_t rawtime;
Willy Tarreau9f66aa92014-05-23 16:36:56 +0200541 static struct tm * timeinfo;
542 static int last_res;
Olivier Burgarde97b9042014-05-22 16:44:59 +0200543
544 d = mo = y = h = m = s = 0;
545 e = field;
546
547 c = *(e++); // remove '['
548 /* day + '/' */
549 while (1) {
550 c = *(e++) - '0';
551 if (c > 9)
552 break;
553 d = d * 10 + c;
554 if (c == (unsigned char)(0 - '0'))
555 goto out_err;
556 }
557
558 /* month + '/' */
559 c = *(e++);
560 if (c =='F') {
561 mo = 2;
562 e = e+3;
563 } else if (c =='S') {
564 mo = 9;
565 e = e+3;
566 } else if (c =='O') {
567 mo = 10;
568 e = e+3;
569 } else if (c =='N') {
570 mo = 11;
571 e = e+3;
572 } else if (c == 'D') {
573 mo = 12;
574 e = e+3;
575 } else if (c == 'A') {
576 c = *(e++);
577 if (c == 'p') {
578 mo = 4;
579 e = e+2;
580 } else if (c == 'u') {
581 mo = 8;
582 e = e+2;
583 } else
584 goto out_err;
585 } else if (c == 'J') {
586 c = *(e++);
587 if (c == 'a') {
588 mo = 1;
589 e = e+2;
590 } else if (c == 'u') {
591 c = *(e++);
592 if (c == 'n') {
593 mo = 6;
594 e = e+1;
595 } else if (c == 'l') {
596 mo = 7;
597 e++;
598 }
599 } else
600 goto out_err;
601 } else if (c == 'M') {
602 e++;
603 c = *(e++);
604 if (c == 'r') {
605 mo = 3;
606 e = e+1;
607 } else if (c == 'y') {
608 mo = 5;
609 e = e+1;
610 } else
611 goto out_err;
612 } else
613 goto out_err;
614
615 /* year + ':' */
616 while (1) {
617 c = *(e++) - '0';
618 if (c > 9)
619 break;
620 y = y * 10 + c;
621 if (c == (unsigned char)(0 - '0'))
622 goto out_err;
623 }
624
625 /* hour + ':' */
Olivier Burgarde97b9042014-05-22 16:44:59 +0200626 while (1) {
627 c = *(e++) - '0';
628 if (c > 9)
629 break;
630 h = h * 10 + c;
631 }
632 if (c == (unsigned char)(0 - '0'))
633 goto out_err;
634
635 /* minute + ':' */
Olivier Burgarde97b9042014-05-22 16:44:59 +0200636 while (1) {
637 c = *(e++) - '0';
638 if (c > 9)
639 break;
640 m = m * 10 + c;
641 }
642 if (c == (unsigned char)(0 - '0'))
643 goto out_err;
644
645 /* second + '.' or ']' */
Olivier Burgarde97b9042014-05-22 16:44:59 +0200646 while (1) {
647 c = *(e++) - '0';
648 if (c > 9)
649 break;
650 s = s * 10 + c;
651 }
652
Willy Tarreau9f66aa92014-05-23 16:36:56 +0200653 if (likely(timeinfo)) {
Willy Tarreau03ca6052020-12-21 08:40:04 +0100654 if ((unsigned)timeinfo->tm_min == m &&
655 (unsigned)timeinfo->tm_hour == h &&
656 (unsigned)timeinfo->tm_mday == d &&
657 (unsigned)timeinfo->tm_mon == mo - 1 &&
658 (unsigned)timeinfo->tm_year == y - 1900)
Willy Tarreau9f66aa92014-05-23 16:36:56 +0200659 return last_res + s;
660 }
661 else {
662 time(&rawtime);
663 timeinfo = localtime(&rawtime);
664 }
Olivier Burgarde97b9042014-05-22 16:44:59 +0200665
Willy Tarreau9f66aa92014-05-23 16:36:56 +0200666 timeinfo->tm_sec = 0;
Olivier Burgarde97b9042014-05-22 16:44:59 +0200667 timeinfo->tm_min = m;
668 timeinfo->tm_hour = h;
669 timeinfo->tm_mday = d;
670 timeinfo->tm_mon = mo - 1;
671 timeinfo->tm_year = y - 1900;
Willy Tarreau9f66aa92014-05-23 16:36:56 +0200672 last_res = mktime(timeinfo);
Olivier Burgarde97b9042014-05-22 16:44:59 +0200673
Willy Tarreau9f66aa92014-05-23 16:36:56 +0200674 return last_res + s;
Olivier Burgarde97b9042014-05-22 16:44:59 +0200675 out_err:
676 return -1;
677}
678
Willy Tarreau72c28532009-01-22 18:56:50 +0100679void truncated_line(int linenum, const char *line)
680{
681 if (!(filter & FILT_QUIET))
682 fprintf(stderr, "Truncated line %d: %s\n", linenum, line);
683}
684
685int main(int argc, char **argv)
686{
Ryan O'Hara8cb99932017-12-15 10:21:39 -0600687 const char *b, *p, *time_field, *accept_field, *source_field;
Hervé COMMOWICK927cddd2011-08-10 17:42:41 +0200688 const char *filter_term_code_name = NULL;
Willy Tarreau72c28532009-01-22 18:56:50 +0100689 const char *output_file = NULL;
Ryan O'Hara8cb99932017-12-15 10:21:39 -0600690 int f, last;
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200691 struct timer *t = NULL;
Willy Tarreau72c28532009-01-22 18:56:50 +0100692 struct eb32_node *n;
Willy Tarreauabe45b62010-10-28 20:33:46 +0200693 struct url_stat *ustat = NULL;
Willy Tarreau72c28532009-01-22 18:56:50 +0100694 int val, test;
Willy Tarreauc8746532014-05-28 23:05:07 +0200695 unsigned int uval;
Willy Tarreau03ca6052020-12-21 08:40:04 +0100696 unsigned int filter_acc_delay = 0, filter_acc_count = 0;
Willy Tarreau5bdfd962009-10-14 15:16:29 +0200697 int filter_time_resp = 0;
Willy Tarreaud3007ff2011-09-05 02:07:23 +0200698 int filt_http_status_low = 0, filt_http_status_high = 0;
Willy Tarreau03ca6052020-12-21 08:40:04 +0100699 unsigned int filt2_timestamp_low = 0, filt2_timestamp_high = 0;
Willy Tarreau72c28532009-01-22 18:56:50 +0100700 int skip_fields = 1;
701
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200702 void (*line_filter)(const char *accept_field, const char *time_field, struct timer **tptr) = NULL;
703
Willy Tarreau72c28532009-01-22 18:56:50 +0100704 argc--; argv++;
705 while (argc > 0) {
706 if (*argv[0] != '-')
707 break;
708
709 if (strcmp(argv[0], "-ad") == 0) {
710 if (argc < 2) die("missing option for -ad");
711 argc--; argv++;
712 filter |= FILT_ACC_DELAY;
713 filter_acc_delay = atol(*argv);
714 }
715 else if (strcmp(argv[0], "-ac") == 0) {
716 if (argc < 2) die("missing option for -ac");
717 argc--; argv++;
718 filter |= FILT_ACC_COUNT;
719 filter_acc_count = atol(*argv);
720 }
Willy Tarreau5bdfd962009-10-14 15:16:29 +0200721 else if (strcmp(argv[0], "-rt") == 0) {
722 if (argc < 2) die("missing option for -rt");
723 argc--; argv++;
724 filter |= FILT_TIME_RESP;
725 filter_time_resp = atol(*argv);
726 }
727 else if (strcmp(argv[0], "-RT") == 0) {
728 if (argc < 2) die("missing option for -RT");
729 argc--; argv++;
730 filter |= FILT_TIME_RESP | FILT_INVERT_TIME_RESP;
731 filter_time_resp = atol(*argv);
732 }
Willy Tarreau72c28532009-01-22 18:56:50 +0100733 else if (strcmp(argv[0], "-s") == 0) {
734 if (argc < 2) die("missing option for -s");
735 argc--; argv++;
736 skip_fields = atol(*argv);
737 }
Willy Tarreau667c9052012-10-10 16:49:28 +0200738 else if (strcmp(argv[0], "-m") == 0) {
739 if (argc < 2) die("missing option for -m");
740 argc--; argv++;
741 lines_max = atol(*argv);
742 }
Willy Tarreau72c28532009-01-22 18:56:50 +0100743 else if (strcmp(argv[0], "-e") == 0)
744 filter |= FILT_ERRORS_ONLY;
Willy Tarreau5bdfd962009-10-14 15:16:29 +0200745 else if (strcmp(argv[0], "-E") == 0)
746 filter |= FILT_ERRORS_ONLY | FILT_INVERT_ERRORS;
Willy Tarreau70c428f2011-07-10 17:27:40 +0200747 else if (strcmp(argv[0], "-H") == 0)
748 filter |= FILT_HTTP_ONLY;
Willy Tarreau08911ff2011-10-13 13:28:36 +0200749 else if (strcmp(argv[0], "-Q") == 0)
750 filter |= FILT_QUEUE_ONLY;
751 else if (strcmp(argv[0], "-QS") == 0)
752 filter |= FILT_QUEUE_SRV_ONLY;
Willy Tarreau72c28532009-01-22 18:56:50 +0100753 else if (strcmp(argv[0], "-c") == 0)
754 filter |= FILT_COUNT_ONLY;
755 else if (strcmp(argv[0], "-q") == 0)
756 filter |= FILT_QUIET;
757 else if (strcmp(argv[0], "-v") == 0)
758 filter_invert = !filter_invert;
759 else if (strcmp(argv[0], "-gt") == 0)
760 filter |= FILT_GRAPH_TIMERS;
Willy Tarreau214c2032009-02-20 11:02:32 +0100761 else if (strcmp(argv[0], "-pct") == 0)
762 filter |= FILT_PERCENTILE;
Willy Tarreau0f423a72010-05-03 10:50:54 +0200763 else if (strcmp(argv[0], "-st") == 0)
764 filter |= FILT_COUNT_STATUS;
Willy Tarreaud2201062010-05-27 18:17:30 +0200765 else if (strcmp(argv[0], "-srv") == 0)
766 filter |= FILT_COUNT_SRV_STATUS;
Willy Tarreau8a09b662012-10-10 10:26:22 +0200767 else if (strcmp(argv[0], "-cc") == 0)
768 filter |= FILT_COUNT_COOK_CODES;
Willy Tarreaud8fc1102010-09-12 17:56:16 +0200769 else if (strcmp(argv[0], "-tc") == 0)
770 filter |= FILT_COUNT_TERM_CODES;
Hervé COMMOWICK927cddd2011-08-10 17:42:41 +0200771 else if (strcmp(argv[0], "-tcn") == 0) {
772 if (argc < 2) die("missing option for -tcn");
773 argc--; argv++;
774 filter |= FILT_TERM_CODE_NAME;
775 filter_term_code_name = *argv;
776 }
777 else if (strcmp(argv[0], "-TCN") == 0) {
778 if (argc < 2) die("missing option for -TCN");
779 argc--; argv++;
780 filter |= FILT_TERM_CODE_NAME | FILT_INVERT_TERM_CODE_NAME;
781 filter_term_code_name = *argv;
782 }
Willy Tarreaud3007ff2011-09-05 02:07:23 +0200783 else if (strcmp(argv[0], "-hs") == 0 || strcmp(argv[0], "-HS") == 0) {
784 char *sep, *str;
785
786 if (argc < 2) die("missing option for -hs/-HS ([min]:[max])");
787 filter |= FILT_HTTP_STATUS;
788 if (argv[0][1] == 'H')
789 filter |= FILT_INVERT_HTTP_STATUS;
790
791 argc--; argv++;
792 str = *argv;
793 sep = strchr(str, ':'); /* [min]:[max] */
794 if (!sep)
795 sep = str; /* make max point to min */
796 else
797 *sep++ = 0;
798 filt_http_status_low = *str ? atol(str) : 0;
799 filt_http_status_high = *sep ? atol(sep) : 65535;
800 }
Olivier Burgarde97b9042014-05-22 16:44:59 +0200801 else if (strcmp(argv[0], "-time") == 0) {
802 char *sep, *str;
803
804 if (argc < 2) die("missing option for -time ([min]:[max])");
805 filter2 |= FILT2_TIMESTAMP;
806
807 argc--; argv++;
808 str = *argv;
809 sep = strchr(str, ':'); /* [min]:[max] */
810 filt2_timestamp_low = *str ? atol(str) : 0;
811 if (!sep)
812 filt2_timestamp_high = 0xFFFFFFFF;
813 else
814 filt2_timestamp_high = atol(++sep);
815 }
Willy Tarreauabe45b62010-10-28 20:33:46 +0200816 else if (strcmp(argv[0], "-u") == 0)
817 filter |= FILT_COUNT_URL_ONLY;
818 else if (strcmp(argv[0], "-uc") == 0)
819 filter |= FILT_COUNT_URL_COUNT;
820 else if (strcmp(argv[0], "-ue") == 0)
821 filter |= FILT_COUNT_URL_ERR;
822 else if (strcmp(argv[0], "-ua") == 0)
823 filter |= FILT_COUNT_URL_TAVG;
824 else if (strcmp(argv[0], "-ut") == 0)
825 filter |= FILT_COUNT_URL_TTOT;
826 else if (strcmp(argv[0], "-uao") == 0)
827 filter |= FILT_COUNT_URL_TAVGO;
828 else if (strcmp(argv[0], "-uto") == 0)
829 filter |= FILT_COUNT_URL_TTOTO;
Baptiste61aaad02012-09-08 23:10:03 +0200830 else if (strcmp(argv[0], "-uba") == 0)
831 filter |= FILT_COUNT_URL_BAVG;
832 else if (strcmp(argv[0], "-ubt") == 0)
833 filter |= FILT_COUNT_URL_BTOT;
Willy Tarreau7cf479c2013-02-16 23:49:04 +0100834 else if (strcmp(argv[0], "-ic") == 0)
835 filter |= FILT_COUNT_IP_COUNT;
Willy Tarreau72c28532009-01-22 18:56:50 +0100836 else if (strcmp(argv[0], "-o") == 0) {
837 if (output_file)
838 die("Fatal: output file name already specified.\n");
839 if (argc < 2)
840 die("Fatal: missing output file name.\n");
841 output_file = argv[1];
842 }
Willy Tarreau615674c2012-01-23 08:15:51 +0100843 else if (strcmp(argv[0], "-h") == 0 || strcmp(argv[0], "--help") == 0)
844 help();
Willy Tarreau72c28532009-01-22 18:56:50 +0100845 argc--;
846 argv++;
847 }
848
849 if (!filter)
850 die("No action specified.\n");
851
852 if (filter & FILT_ACC_COUNT && !filter_acc_count)
853 filter_acc_count=1;
854
855 if (filter & FILT_ACC_DELAY && !filter_acc_delay)
856 filter_acc_delay = 1;
857
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200858
859 /* by default, all lines are printed */
860 line_filter = filter_output_line;
861 if (filter & (FILT_ACC_COUNT|FILT_ACC_DELAY))
862 line_filter = filter_accept_holes;
863 else if (filter & (FILT_GRAPH_TIMERS|FILT_PERCENTILE))
864 line_filter = filter_graphs;
865 else if (filter & FILT_COUNT_STATUS)
866 line_filter = filter_count_status;
Willy Tarreau8a09b662012-10-10 10:26:22 +0200867 else if (filter & FILT_COUNT_COOK_CODES)
868 line_filter = filter_count_cook_codes;
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200869 else if (filter & FILT_COUNT_TERM_CODES)
870 line_filter = filter_count_term_codes;
871 else if (filter & FILT_COUNT_SRV_STATUS)
872 line_filter = filter_count_srv_status;
873 else if (filter & FILT_COUNT_URL_ANY)
874 line_filter = filter_count_url;
875 else if (filter & FILT_COUNT_ONLY)
876 line_filter = NULL;
Willy Tarreau72c28532009-01-22 18:56:50 +0100877
Willy Tarreauf8c95d22012-06-12 09:16:56 +0200878#if defined(POSIX_FADV_SEQUENTIAL)
879 /* around 20% performance improvement is observed on Linux with this
Joseph Herlant42172bd2018-11-09 18:02:35 -0800880 * on cold-cache. Surprisingly, WILLNEED is less performant. Don't
Willy Tarreauf8c95d22012-06-12 09:16:56 +0200881 * use NOREUSE as it flushes the cache and prevents easy data
882 * manipulation on logs!
883 */
884 posix_fadvise(0, 0, 0, POSIX_FADV_SEQUENTIAL);
885#endif
886
Willy Tarreaua1629a52012-11-13 20:48:15 +0100887 if (!line_filter && /* FILT_COUNT_ONLY ( see above), and no input filter (see below) */
Olivier Burgarde97b9042014-05-22 16:44:59 +0200888 !(filter & (FILT_HTTP_ONLY|FILT_TIME_RESP|FILT_ERRORS_ONLY|FILT_HTTP_STATUS|FILT_QUEUE_ONLY|FILT_QUEUE_SRV_ONLY|FILT_TERM_CODE_NAME)) &&
889 !(filter2 & (FILT2_TIMESTAMP))) {
Willy Tarreaua1629a52012-11-13 20:48:15 +0100890 /* read the whole file at once first, ignore it if inverted output */
Willy Tarreaue1a908c2012-01-03 09:23:03 +0100891 if (!filter_invert)
Willy Tarreaua1629a52012-11-13 20:48:15 +0100892 while ((lines_max < 0 || lines_out < lines_max) && fgets2(stdin) != NULL)
Willy Tarreaue1a908c2012-01-03 09:23:03 +0100893 lines_out++;
894
895 goto skip_filters;
896 }
897
Willy Tarreau214c2032009-02-20 11:02:32 +0100898 while ((line = fgets2(stdin)) != NULL) {
Willy Tarreau72c28532009-01-22 18:56:50 +0100899 linenum++;
Willy Tarreau26deaf52011-07-10 19:47:48 +0200900 time_field = NULL; accept_field = NULL;
Willy Tarreau7cf479c2013-02-16 23:49:04 +0100901 source_field = NULL;
Willy Tarreau72c28532009-01-22 18:56:50 +0100902
Willy Tarreau5bdfd962009-10-14 15:16:29 +0200903 test = 1;
Willy Tarreau26deaf52011-07-10 19:47:48 +0200904
905 /* for any line we process, we first ensure that there is a field
906 * looking like the accept date field (beginning with a '[').
907 */
Willy Tarreau7cf479c2013-02-16 23:49:04 +0100908 if (filter & FILT_COUNT_IP_COUNT) {
909 /* we need the IP first */
910 source_field = field_start(line, SOURCE_FIELD + skip_fields);
911 accept_field = field_start(source_field, ACCEPT_FIELD - SOURCE_FIELD + 1);
912 }
913 else
914 accept_field = field_start(line, ACCEPT_FIELD + skip_fields);
915
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200916 if (unlikely(*accept_field != '[')) {
Willy Tarreau26deaf52011-07-10 19:47:48 +0200917 parse_err++;
918 continue;
919 }
920
921 /* the day of month field is begin 01 and 31 */
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200922 if (accept_field[1] < '0' || accept_field[1] > '3') {
Willy Tarreau26deaf52011-07-10 19:47:48 +0200923 parse_err++;
924 continue;
925 }
926
Olivier Burgarde97b9042014-05-22 16:44:59 +0200927 if (filter2 & FILT2_TIMESTAMP) {
928 uval = convert_date_to_timestamp(accept_field);
929 test &= (uval>=filt2_timestamp_low && uval<=filt2_timestamp_high) ;
930 }
931
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200932 if (filter & FILT_HTTP_ONLY) {
Willy Tarreau70c428f2011-07-10 17:27:40 +0200933 /* only report lines with at least 4 timers */
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200934 if (!time_field) {
Willy Tarreau26deaf52011-07-10 19:47:48 +0200935 time_field = field_start(accept_field, TIME_FIELD - ACCEPT_FIELD + 1);
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200936 if (unlikely(!*time_field)) {
Willy Tarreau26deaf52011-07-10 19:47:48 +0200937 truncated_line(linenum, line);
938 continue;
939 }
Willy Tarreau70c428f2011-07-10 17:27:40 +0200940 }
941
Ryan O'Hara8cb99932017-12-15 10:21:39 -0600942 field_stop(time_field + 1);
Willy Tarreau758a6ea2011-07-10 18:53:44 +0200943 /* we have field TIME_FIELD in [time_field]..[e-1] */
944 p = time_field;
Willy Tarreau70c428f2011-07-10 17:27:40 +0200945 f = 0;
Willy Tarreaudf6f0d12011-07-10 18:15:08 +0200946 while (!SEP(*p)) {
Willy Tarreau70c428f2011-07-10 17:27:40 +0200947 if (++f == 4)
948 break;
949 SKIP_CHAR(p, '/');
950 }
951 test &= (f >= 4);
952 }
953
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200954 if (filter & FILT_TIME_RESP) {
Willy Tarreau5bdfd962009-10-14 15:16:29 +0200955 int tps;
956
957 /* only report lines with response times larger than filter_time_resp */
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200958 if (!time_field) {
Willy Tarreau26deaf52011-07-10 19:47:48 +0200959 time_field = field_start(accept_field, TIME_FIELD - ACCEPT_FIELD + 1);
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200960 if (unlikely(!*time_field)) {
Willy Tarreau26deaf52011-07-10 19:47:48 +0200961 truncated_line(linenum, line);
962 continue;
963 }
Willy Tarreau5bdfd962009-10-14 15:16:29 +0200964 }
965
Ryan O'Hara8cb99932017-12-15 10:21:39 -0600966 field_stop(time_field + 1);
Willy Tarreau758a6ea2011-07-10 18:53:44 +0200967 /* we have field TIME_FIELD in [time_field]..[e-1], let's check only the response time */
Willy Tarreau5bdfd962009-10-14 15:16:29 +0200968
Willy Tarreau758a6ea2011-07-10 18:53:44 +0200969 p = time_field;
Willy Tarreau24bcb4f2010-10-28 20:39:50 +0200970 f = 0;
Willy Tarreaudf6f0d12011-07-10 18:15:08 +0200971 while (!SEP(*p)) {
Willy Tarreau5bdfd962009-10-14 15:16:29 +0200972 tps = str2ic(p);
973 if (tps < 0) {
974 tps = -1;
Willy Tarreau5bdfd962009-10-14 15:16:29 +0200975 }
Willy Tarreau24bcb4f2010-10-28 20:39:50 +0200976 if (++f == 4)
977 break;
Willy Tarreau5bdfd962009-10-14 15:16:29 +0200978 SKIP_CHAR(p, '/');
979 }
980
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200981 if (unlikely(f < 4)) {
Willy Tarreau5bdfd962009-10-14 15:16:29 +0200982 parse_err++;
983 continue;
984 }
985
986 test &= (tps >= filter_time_resp) ^ !!(filter & FILT_INVERT_TIME_RESP);
987 }
988
Willy Tarreaud3007ff2011-09-05 02:07:23 +0200989 if (filter & (FILT_ERRORS_ONLY | FILT_HTTP_STATUS)) {
990 /* Check both error codes (-1, 5xx) and status code ranges */
Willy Tarreau26deaf52011-07-10 19:47:48 +0200991 if (time_field)
992 b = field_start(time_field, STATUS_FIELD - TIME_FIELD + 1);
993 else
994 b = field_start(accept_field, STATUS_FIELD - ACCEPT_FIELD + 1);
995
Willy Tarreaua2b39fb2011-07-10 21:39:35 +0200996 if (unlikely(!*b)) {
Willy Tarreau72c28532009-01-22 18:56:50 +0100997 truncated_line(linenum, line);
998 continue;
999 }
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001000
Willy Tarreaud3007ff2011-09-05 02:07:23 +02001001 val = str2ic(b);
1002 if (filter & FILT_ERRORS_ONLY)
1003 test &= (val < 0 || (val >= 500 && val <= 599)) ^ !!(filter & FILT_INVERT_ERRORS);
1004
1005 if (filter & FILT_HTTP_STATUS)
1006 test &= (val >= filt_http_status_low && val <= filt_http_status_high) ^ !!(filter & FILT_INVERT_HTTP_STATUS);
Willy Tarreau72c28532009-01-22 18:56:50 +01001007 }
1008
Willy Tarreau08911ff2011-10-13 13:28:36 +02001009 if (filter & (FILT_QUEUE_ONLY|FILT_QUEUE_SRV_ONLY)) {
1010 /* Check if the server's queue is non-nul */
1011 if (time_field)
1012 b = field_start(time_field, QUEUE_LEN_FIELD - TIME_FIELD + 1);
1013 else
1014 b = field_start(accept_field, QUEUE_LEN_FIELD - ACCEPT_FIELD + 1);
1015
1016 if (unlikely(!*b)) {
1017 truncated_line(linenum, line);
1018 continue;
1019 }
1020
1021 if (*b == '0') {
1022 if (filter & FILT_QUEUE_SRV_ONLY) {
1023 test = 0;
1024 }
1025 else {
1026 do {
1027 b++;
1028 if (*b == '/') {
1029 b++;
1030 break;
1031 }
1032 } while (*b);
1033 test &= ((unsigned char)(*b - '1') < 9);
1034 }
1035 }
1036 }
1037
Hervé COMMOWICK927cddd2011-08-10 17:42:41 +02001038 if (filter & FILT_TERM_CODE_NAME) {
1039 /* only report corresponding termination code name */
1040 if (time_field)
1041 b = field_start(time_field, TERM_CODES_FIELD - TIME_FIELD + 1);
1042 else
1043 b = field_start(accept_field, TERM_CODES_FIELD - ACCEPT_FIELD + 1);
1044
1045 if (unlikely(!*b)) {
1046 truncated_line(linenum, line);
1047 continue;
1048 }
1049
1050 test &= (b[0] == filter_term_code_name[0] && b[1] == filter_term_code_name[1]) ^ !!(filter & FILT_INVERT_TERM_CODE_NAME);
1051 }
1052
1053
Willy Tarreau0f423a72010-05-03 10:50:54 +02001054 test ^= filter_invert;
1055 if (!test)
1056 continue;
1057
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001058 /************** here we process inputs *******************/
Willy Tarreau72c28532009-01-22 18:56:50 +01001059
Willy Tarreau7cf479c2013-02-16 23:49:04 +01001060 if (line_filter) {
1061 if (filter & FILT_COUNT_IP_COUNT)
1062 filter_count_ip(source_field, accept_field, time_field, &t);
1063 else
1064 line_filter(accept_field, time_field, &t);
1065 }
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001066 else
Willy Tarreaua1629a52012-11-13 20:48:15 +01001067 lines_out++; /* FILT_COUNT_ONLY was used, so we're just counting lines */
1068 if (lines_max >= 0 && lines_out >= lines_max)
Willy Tarreau667c9052012-10-10 16:49:28 +02001069 break;
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001070 }
Willy Tarreauabe45b62010-10-28 20:33:46 +02001071
Willy Tarreaue1a908c2012-01-03 09:23:03 +01001072 skip_filters:
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001073 /*****************************************************
1074 * Here we've finished reading all input. Depending on the
1075 * filters, we may still have some analysis to run on the
1076 * collected data and to output data in a new format.
1077 *************************************************** */
Willy Tarreau72c28532009-01-22 18:56:50 +01001078
1079 if (t)
1080 free(t);
1081
1082 if (filter & FILT_COUNT_ONLY) {
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001083 printf("%d\n", lines_out);
Willy Tarreau72c28532009-01-22 18:56:50 +01001084 exit(0);
1085 }
1086
Willy Tarreau72c28532009-01-22 18:56:50 +01001087 if (filter & (FILT_ACC_COUNT|FILT_ACC_DELAY)) {
1088 /* sort and count all timers. Output will look like this :
1089 * <accept_date> <delta_ms from previous one> <nb entries>
1090 */
1091 n = eb32_first(&timers[0]);
1092
1093 if (n)
1094 last = n->key;
1095 while (n) {
1096 unsigned int d, h, m, s, ms;
1097
1098 t = container_of(n, struct timer, node);
1099 h = n->key;
1100 d = h - last;
1101 last = h;
1102
1103 if (d >= filter_acc_delay && t->count >= filter_acc_count) {
1104 ms = h % 1000; h = h / 1000;
1105 s = h % 60; h = h / 60;
1106 m = h % 60; h = h / 60;
Willy Tarreau72c28532009-01-22 18:56:50 +01001107 printf("%02d:%02d:%02d.%03d %d %d %d\n", h, m, s, ms, last, d, t->count);
Willy Tarreau667c9052012-10-10 16:49:28 +02001108 lines_out++;
Willy Tarreaua1629a52012-11-13 20:48:15 +01001109 if (lines_max >= 0 && lines_out >= lines_max)
Willy Tarreau667c9052012-10-10 16:49:28 +02001110 break;
Willy Tarreau72c28532009-01-22 18:56:50 +01001111 }
1112 n = eb32_next(n);
1113 }
1114 }
1115 else if (filter & FILT_GRAPH_TIMERS) {
1116 /* sort all timers */
1117 for (f = 0; f < 5; f++) {
1118 struct eb32_node *n;
1119 int val;
1120
1121 val = 0;
1122 n = eb32_first(&timers[f]);
1123 while (n) {
1124 int i;
1125 double d;
1126
1127 t = container_of(n, struct timer, node);
1128 last = n->key;
1129 val = t->count;
1130
1131 i = (last < 0) ? -last : last;
1132 i = fls_auto(i) - QBITS;
1133
1134 if (i > 0)
1135 d = val / (double)(1 << i);
1136 else
1137 d = val;
1138
Willy Tarreaua1629a52012-11-13 20:48:15 +01001139 if (d > 0.0)
Willy Tarreau72c28532009-01-22 18:56:50 +01001140 printf("%d %d %f\n", f, last, d+1.0);
Willy Tarreau72c28532009-01-22 18:56:50 +01001141
1142 n = eb32_next(n);
1143 }
Willy Tarreau214c2032009-02-20 11:02:32 +01001144 }
1145 }
1146 else if (filter & FILT_PERCENTILE) {
1147 /* report timers by percentile :
1148 * <percent> <total> <max_req_time> <max_conn_time> <max_resp_time> <max_data_time>
1149 * We don't count errs.
1150 */
1151 struct eb32_node *n[5];
1152 unsigned long cum[5];
1153 double step;
1154
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001155 if (!lines_out)
Willy Tarreau910ba4b2009-11-17 10:16:19 +01001156 goto empty;
1157
Willy Tarreau214c2032009-02-20 11:02:32 +01001158 for (f = 1; f < 5; f++) {
1159 n[f] = eb32_first(&timers[f]);
1160 cum[f] = container_of(n[f], struct timer, node)->count;
1161 }
1162
1163 for (step = 1; step <= 1000;) {
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001164 unsigned int thres = lines_out * (step / 1000.0);
Willy Tarreau214c2032009-02-20 11:02:32 +01001165
1166 printf("%3.1f %d ", step/10.0, thres);
1167 for (f = 1; f < 5; f++) {
1168 struct eb32_node *next;
1169 while (cum[f] < thres) {
1170 /* need to find other keys */
1171 next = eb32_next(n[f]);
1172 if (!next)
1173 break;
1174 n[f] = next;
1175 cum[f] += container_of(next, struct timer, node)->count;
1176 }
1177
1178 /* value still within $step % of total */
1179 printf("%d ", n[f]->key);
1180 }
1181 putchar('\n');
1182 if (step >= 100 && step < 900)
1183 step += 50; // jump 5% by 5% between those steps.
1184 else if (step >= 20 && step < 980)
1185 step += 10;
1186 else
1187 step += 1;
Willy Tarreau72c28532009-01-22 18:56:50 +01001188 }
1189 }
Willy Tarreau0f423a72010-05-03 10:50:54 +02001190 else if (filter & FILT_COUNT_STATUS) {
1191 /* output all statuses in the form of <status> <occurrences> */
1192 n = eb32_first(&timers[0]);
1193 while (n) {
1194 t = container_of(n, struct timer, node);
1195 printf("%d %d\n", n->key, t->count);
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001196 lines_out++;
Willy Tarreaua1629a52012-11-13 20:48:15 +01001197 if (lines_max >= 0 && lines_out >= lines_max)
Willy Tarreau667c9052012-10-10 16:49:28 +02001198 break;
Willy Tarreau0f423a72010-05-03 10:50:54 +02001199 n = eb32_next(n);
1200 }
1201 }
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001202 else if (filter & FILT_COUNT_SRV_STATUS) {
Willy Tarreaud2201062010-05-27 18:17:30 +02001203 struct ebmb_node *srv_node;
1204 struct srv_st *srv;
1205
1206 printf("#srv_name 1xx 2xx 3xx 4xx 5xx other tot_req req_ok pct_ok avg_ct avg_rt\n");
1207
1208 srv_node = ebmb_first(&timers[0]);
1209 while (srv_node) {
1210 int tot_rq;
1211
1212 srv = container_of(srv_node, struct srv_st, node);
1213
1214 tot_rq = 0;
1215 for (f = 0; f <= 5; f++)
1216 tot_rq += srv->st_cnt[f];
1217
1218 printf("%s %d %d %d %d %d %d %d %d %.1f %d %d\n",
1219 srv_node->key, srv->st_cnt[1], srv->st_cnt[2],
1220 srv->st_cnt[3], srv->st_cnt[4], srv->st_cnt[5], srv->st_cnt[0],
1221 tot_rq,
1222 srv->nb_ok, (double)srv->nb_ok * 100.0 / (tot_rq?tot_rq:1),
1223 (int)(srv->cum_ct / (srv->nb_ct?srv->nb_ct:1)), (int)(srv->cum_rt / (srv->nb_rt?srv->nb_rt:1)));
1224 srv_node = ebmb_next(srv_node);
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001225 lines_out++;
Willy Tarreaua1629a52012-11-13 20:48:15 +01001226 if (lines_max >= 0 && lines_out >= lines_max)
Willy Tarreau667c9052012-10-10 16:49:28 +02001227 break;
Willy Tarreaud2201062010-05-27 18:17:30 +02001228 }
1229 }
Willy Tarreau8a09b662012-10-10 10:26:22 +02001230 else if (filter & (FILT_COUNT_TERM_CODES|FILT_COUNT_COOK_CODES)) {
Willy Tarreaud8fc1102010-09-12 17:56:16 +02001231 /* output all statuses in the form of <code> <occurrences> */
1232 n = eb32_first(&timers[0]);
1233 while (n) {
1234 t = container_of(n, struct timer, node);
1235 printf("%c%c %d\n", (n->key >> 8), (n->key) & 255, t->count);
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001236 lines_out++;
Willy Tarreaua1629a52012-11-13 20:48:15 +01001237 if (lines_max >= 0 && lines_out >= lines_max)
Willy Tarreau667c9052012-10-10 16:49:28 +02001238 break;
Willy Tarreaud8fc1102010-09-12 17:56:16 +02001239 n = eb32_next(n);
1240 }
1241 }
Willy Tarreau7cf479c2013-02-16 23:49:04 +01001242 else if (filter & (FILT_COUNT_URL_ANY|FILT_COUNT_IP_COUNT)) {
Willy Tarreauabe45b62010-10-28 20:33:46 +02001243 struct eb_node *node, *next;
1244
1245 if (!(filter & FILT_COUNT_URL_ONLY)) {
1246 /* we have to sort on another criterion. We'll use timers[1] for the
1247 * destination tree.
1248 */
1249
1250 timers[1] = EB_ROOT; /* reconfigure to accept duplicates */
1251 for (node = eb_first(&timers[0]); node; node = next) {
1252 next = eb_next(node);
1253 eb_delete(node);
1254
1255 ustat = container_of(node, struct url_stat, node.url.node);
1256
Willy Tarreau7cf479c2013-02-16 23:49:04 +01001257 if (filter & (FILT_COUNT_URL_COUNT|FILT_COUNT_IP_COUNT))
Willy Tarreauabe45b62010-10-28 20:33:46 +02001258 ustat->node.val.key = ustat->nb_req;
1259 else if (filter & FILT_COUNT_URL_ERR)
1260 ustat->node.val.key = ustat->nb_err;
1261 else if (filter & FILT_COUNT_URL_TTOT)
1262 ustat->node.val.key = ustat->total_time;
1263 else if (filter & FILT_COUNT_URL_TAVG)
1264 ustat->node.val.key = ustat->nb_req ? ustat->total_time / ustat->nb_req : 0;
1265 else if (filter & FILT_COUNT_URL_TTOTO)
1266 ustat->node.val.key = ustat->total_time_ok;
1267 else if (filter & FILT_COUNT_URL_TAVGO)
1268 ustat->node.val.key = (ustat->nb_req - ustat->nb_err) ? ustat->total_time_ok / (ustat->nb_req - ustat->nb_err) : 0;
Baptiste61aaad02012-09-08 23:10:03 +02001269 else if (filter & FILT_COUNT_URL_BAVG)
1270 ustat->node.val.key = ustat->nb_req ? ustat->total_bytes_sent / ustat->nb_req : 0;
1271 else if (filter & FILT_COUNT_URL_BTOT)
1272 ustat->node.val.key = ustat->total_bytes_sent;
Willy Tarreauabe45b62010-10-28 20:33:46 +02001273 else
1274 ustat->node.val.key = 0;
1275
1276 eb64_insert(&timers[1], &ustat->node.val);
1277 }
1278 /* switch trees */
1279 timers[0] = timers[1];
1280 }
1281
Willy Tarreau7cf479c2013-02-16 23:49:04 +01001282 if (FILT_COUNT_IP_COUNT)
1283 printf("#req err ttot tavg oktot okavg bavg btot src\n");
1284 else
1285 printf("#req err ttot tavg oktot okavg bavg btot url\n");
Willy Tarreauabe45b62010-10-28 20:33:46 +02001286
1287 /* scan the tree in its reverse sorting order */
1288 node = eb_last(&timers[0]);
1289 while (node) {
1290 ustat = container_of(node, struct url_stat, node.url.node);
Willy Tarreau2df860c2020-12-21 08:29:09 +01001291 printf("%d %d %llu %llu %llu %llu %llu %llu %s\n",
Willy Tarreauabe45b62010-10-28 20:33:46 +02001292 ustat->nb_req,
1293 ustat->nb_err,
1294 ustat->total_time,
1295 ustat->nb_req ? ustat->total_time / ustat->nb_req : 0,
1296 ustat->total_time_ok,
1297 (ustat->nb_req - ustat->nb_err) ? ustat->total_time_ok / (ustat->nb_req - ustat->nb_err) : 0,
Baptiste61aaad02012-09-08 23:10:03 +02001298 ustat->nb_req ? ustat->total_bytes_sent / ustat->nb_req : 0,
1299 ustat->total_bytes_sent,
Willy Tarreauabe45b62010-10-28 20:33:46 +02001300 ustat->url);
1301
1302 node = eb_prev(node);
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001303 lines_out++;
Willy Tarreaua1629a52012-11-13 20:48:15 +01001304 if (lines_max >= 0 && lines_out >= lines_max)
Willy Tarreau667c9052012-10-10 16:49:28 +02001305 break;
Willy Tarreauabe45b62010-10-28 20:33:46 +02001306 }
1307 }
Willy Tarreaud2201062010-05-27 18:17:30 +02001308
Willy Tarreau910ba4b2009-11-17 10:16:19 +01001309 empty:
Willy Tarreau72c28532009-01-22 18:56:50 +01001310 if (!(filter & FILT_QUIET))
1311 fprintf(stderr, "%d lines in, %d lines out, %d parsing errors\n",
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001312 linenum, lines_out, parse_err);
Willy Tarreau72c28532009-01-22 18:56:50 +01001313 exit(0);
1314}
1315
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001316void filter_output_line(const char *accept_field, const char *time_field, struct timer **tptr)
1317{
1318 puts(line);
1319 lines_out++;
1320}
1321
1322void filter_accept_holes(const char *accept_field, const char *time_field, struct timer **tptr)
1323{
1324 struct timer *t2;
1325 int val;
1326
1327 val = convert_date(accept_field);
1328 if (unlikely(val < 0)) {
1329 truncated_line(linenum, line);
1330 return;
1331 }
1332
1333 t2 = insert_value(&timers[0], tptr, val);
1334 t2->count++;
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001335 return;
1336}
1337
1338void filter_count_status(const char *accept_field, const char *time_field, struct timer **tptr)
1339{
1340 struct timer *t2;
1341 const char *b;
1342 int val;
1343
1344 if (time_field)
1345 b = field_start(time_field, STATUS_FIELD - TIME_FIELD + 1);
1346 else
1347 b = field_start(accept_field, STATUS_FIELD - ACCEPT_FIELD + 1);
1348
1349 if (unlikely(!*b)) {
1350 truncated_line(linenum, line);
1351 return;
1352 }
1353
1354 val = str2ic(b);
1355
1356 t2 = insert_value(&timers[0], tptr, val);
1357 t2->count++;
1358}
1359
Willy Tarreau8a09b662012-10-10 10:26:22 +02001360void filter_count_cook_codes(const char *accept_field, const char *time_field, struct timer **tptr)
1361{
1362 struct timer *t2;
1363 const char *b;
1364 int val;
1365
1366 if (time_field)
1367 b = field_start(time_field, TERM_CODES_FIELD - TIME_FIELD + 1);
1368 else
1369 b = field_start(accept_field, TERM_CODES_FIELD - ACCEPT_FIELD + 1);
1370
1371 if (unlikely(!*b)) {
1372 truncated_line(linenum, line);
1373 return;
1374 }
1375
1376 val = 256 * b[2] + b[3];
1377
1378 t2 = insert_value(&timers[0], tptr, val);
1379 t2->count++;
1380}
1381
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001382void filter_count_term_codes(const char *accept_field, const char *time_field, struct timer **tptr)
1383{
1384 struct timer *t2;
1385 const char *b;
1386 int val;
1387
1388 if (time_field)
1389 b = field_start(time_field, TERM_CODES_FIELD - TIME_FIELD + 1);
1390 else
1391 b = field_start(accept_field, TERM_CODES_FIELD - ACCEPT_FIELD + 1);
1392
1393 if (unlikely(!*b)) {
1394 truncated_line(linenum, line);
1395 return;
1396 }
1397
1398 val = 256 * b[0] + b[1];
1399
1400 t2 = insert_value(&timers[0], tptr, val);
1401 t2->count++;
1402}
1403
1404void filter_count_srv_status(const char *accept_field, const char *time_field, struct timer **tptr)
1405{
1406 const char *b, *e, *p;
1407 int f, err, array[5];
1408 struct ebmb_node *srv_node;
1409 struct srv_st *srv;
1410 int val;
1411
1412 /* the server field is before the status field, so let's
1413 * parse them in the proper order.
1414 */
1415 b = field_start(accept_field, SERVER_FIELD - ACCEPT_FIELD + 1);
1416 if (unlikely(!*b)) {
1417 truncated_line(linenum, line);
1418 return;
1419 }
1420
1421 e = field_stop(b + 1); /* we have the server name in [b]..[e-1] */
1422
1423 /* the chance that a server name already exists is extremely high,
1424 * so let's perform a normal lookup first.
1425 */
1426 srv_node = ebst_lookup_len(&timers[0], b, e - b);
1427 srv = container_of(srv_node, struct srv_st, node);
1428
1429 if (!srv_node) {
1430 /* server not yet in the tree, let's create it */
1431 srv = (void *)calloc(1, sizeof(struct srv_st) + e - b + 1);
1432 srv_node = &srv->node;
1433 memcpy(&srv_node->key, b, e - b);
1434 srv_node->key[e - b] = '\0';
1435 ebst_insert(&timers[0], srv_node);
1436 }
1437
1438 /* let's collect the connect and response times */
1439 if (!time_field) {
1440 time_field = field_start(e, TIME_FIELD - SERVER_FIELD);
1441 if (unlikely(!*time_field)) {
1442 truncated_line(linenum, line);
1443 return;
1444 }
1445 }
1446
1447 e = field_stop(time_field + 1);
1448 /* we have field TIME_FIELD in [time_field]..[e-1] */
1449
1450 p = time_field;
1451 err = 0;
1452 f = 0;
1453 while (!SEP(*p)) {
1454 array[f] = str2ic(p);
1455 if (array[f] < 0) {
1456 array[f] = -1;
1457 err = 1;
1458 }
1459 if (++f == 5)
1460 break;
1461 SKIP_CHAR(p, '/');
1462 }
1463
1464 if (unlikely(f < 5)){
1465 parse_err++;
1466 return;
1467 }
1468
1469 /* OK we have our timers in array[2,3] */
1470 if (!err)
1471 srv->nb_ok++;
1472
1473 if (array[2] >= 0) {
1474 srv->cum_ct += array[2];
1475 srv->nb_ct++;
1476 }
1477
1478 if (array[3] >= 0) {
1479 srv->cum_rt += array[3];
1480 srv->nb_rt++;
1481 }
1482
1483 /* we're interested in the 5 HTTP status classes (1xx ... 5xx), and
1484 * the invalid ones which will be reported as 0.
1485 */
1486 b = field_start(e, STATUS_FIELD - TIME_FIELD);
1487 if (unlikely(!*b)) {
1488 truncated_line(linenum, line);
1489 return;
1490 }
1491
1492 val = 0;
1493 if (*b >= '1' && *b <= '5')
1494 val = *b - '0';
1495
1496 srv->st_cnt[val]++;
1497}
1498
1499void filter_count_url(const char *accept_field, const char *time_field, struct timer **tptr)
1500{
1501 struct url_stat *ustat = NULL;
1502 struct ebpt_node *ebpt_old;
1503 const char *b, *e;
1504 int f, err, array[5];
Baptiste61aaad02012-09-08 23:10:03 +02001505 int val;
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001506
1507 /* let's collect the response time */
1508 if (!time_field) {
1509 time_field = field_start(accept_field, TIME_FIELD - ACCEPT_FIELD + 1); // avg 115 ns per line
1510 if (unlikely(!*time_field)) {
1511 truncated_line(linenum, line);
1512 return;
1513 }
1514 }
1515
1516 /* we have the field TIME_FIELD starting at <time_field>. We'll
1517 * parse the 5 timers to detect errors, it takes avg 55 ns per line.
1518 */
1519 e = time_field; err = 0; f = 0;
1520 while (!SEP(*e)) {
1521 array[f] = str2ic(e);
1522 if (array[f] < 0) {
1523 array[f] = -1;
1524 err = 1;
1525 }
1526 if (++f == 5)
1527 break;
1528 SKIP_CHAR(e, '/');
1529 }
1530 if (f < 5) {
1531 parse_err++;
1532 return;
1533 }
1534
1535 /* OK we have our timers in array[3], and err is >0 if at
1536 * least one -1 was seen. <e> points to the first char of
1537 * the last timer. Let's prepare a new node with that.
1538 */
1539 if (unlikely(!ustat))
1540 ustat = calloc(1, sizeof(*ustat));
1541
1542 ustat->nb_err = err;
1543 ustat->nb_req = 1;
1544
1545 /* use array[4] = total time in case of error */
1546 ustat->total_time = (array[3] >= 0) ? array[3] : array[4];
1547 ustat->total_time_ok = (array[3] >= 0) ? array[3] : 0;
1548
Baptiste61aaad02012-09-08 23:10:03 +02001549 e = field_start(e, BYTES_SENT_FIELD - TIME_FIELD + 1);
1550 val = str2ic(e);
1551 ustat->total_bytes_sent = val;
1552
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001553 /* the line may be truncated because of a bad request or anything like this,
1554 * without a method. Also, if it does not begin with an quote, let's skip to
1555 * the next field because it's a capture. Let's fall back to the "method" itself
1556 * if there's nothing else.
1557 */
Baptiste61aaad02012-09-08 23:10:03 +02001558 e = field_start(e, METH_FIELD - BYTES_SENT_FIELD + 1);
Willy Tarreau61a40c72011-09-06 08:11:27 +02001559 while (*e != '"' && *e) {
1560 /* Note: some syslog servers escape quotes ! */
1561 if (*e == '\\' && e[1] == '"')
1562 break;
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001563 e = field_start(e, 2);
Willy Tarreau61a40c72011-09-06 08:11:27 +02001564 }
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001565
1566 if (unlikely(!*e)) {
1567 truncated_line(linenum, line);
Ilya Shipitsin4473a2e2017-09-22 22:33:16 +05001568 free(ustat);
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001569 return;
1570 }
1571
1572 b = field_start(e, URL_FIELD - METH_FIELD + 1); // avg 40 ns per line
1573 if (!*b)
1574 b = e;
1575
1576 /* stop at end of field or first ';' or '?', takes avg 64 ns per line */
1577 e = b;
1578 do {
Willy Tarreau14389e72011-07-10 22:11:17 +02001579 if (*e == ' ' || *e == '?' || *e == ';') {
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001580 *(char *)e = 0;
1581 break;
1582 }
1583 e++;
1584 } while (*e);
1585
1586 /* now instead of copying the URL for a simple lookup, we'll link
1587 * to it from the node we're trying to insert. If it returns a
1588 * different value, it was already there. Otherwise we just have
1589 * to dynamically realloc an entry using strdup().
1590 */
1591 ustat->node.url.key = (char *)b;
1592 ebpt_old = ebis_insert(&timers[0], &ustat->node.url);
1593
1594 if (ebpt_old != &ustat->node.url) {
1595 struct url_stat *ustat_old;
1596 /* node was already there, let's update previous one */
1597 ustat_old = container_of(ebpt_old, struct url_stat, node.url);
1598 ustat_old->nb_req ++;
1599 ustat_old->nb_err += ustat->nb_err;
1600 ustat_old->total_time += ustat->total_time;
1601 ustat_old->total_time_ok += ustat->total_time_ok;
Baptiste61aaad02012-09-08 23:10:03 +02001602 ustat_old->total_bytes_sent += ustat->total_bytes_sent;
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001603 } else {
1604 ustat->url = ustat->node.url.key = strdup(ustat->node.url.key);
1605 ustat = NULL; /* node was used */
1606 }
1607}
1608
Willy Tarreau7cf479c2013-02-16 23:49:04 +01001609void filter_count_ip(const char *source_field, const char *accept_field, const char *time_field, struct timer **tptr)
1610{
1611 struct url_stat *ustat = NULL;
1612 struct ebpt_node *ebpt_old;
1613 const char *b, *e;
1614 int f, err, array[5];
1615 int val;
1616
1617 /* let's collect the response time */
1618 if (!time_field) {
1619 time_field = field_start(accept_field, TIME_FIELD - ACCEPT_FIELD + 1); // avg 115 ns per line
1620 if (unlikely(!*time_field)) {
1621 truncated_line(linenum, line);
1622 return;
1623 }
1624 }
1625
1626 /* we have the field TIME_FIELD starting at <time_field>. We'll
1627 * parse the 5 timers to detect errors, it takes avg 55 ns per line.
1628 */
1629 e = time_field; err = 0; f = 0;
1630 while (!SEP(*e)) {
1631 if (f == 0 || f == 4) {
1632 array[f] = str2ic(e);
1633 if (array[f] < 0) {
1634 array[f] = -1;
1635 err = 1;
1636 }
1637 }
1638 if (++f == 5)
1639 break;
1640 SKIP_CHAR(e, '/');
1641 }
1642 if (f < 5) {
1643 parse_err++;
1644 return;
1645 }
1646
1647 /* OK we have our timers in array[0], and err is >0 if at
1648 * least one -1 was seen. <e> points to the first char of
1649 * the last timer. Let's prepare a new node with that.
1650 */
1651 if (unlikely(!ustat))
1652 ustat = calloc(1, sizeof(*ustat));
1653
1654 ustat->nb_err = err;
1655 ustat->nb_req = 1;
1656
1657 /* use array[4] = total time in case of error */
1658 ustat->total_time = (array[0] >= 0) ? array[0] : array[4];
1659 ustat->total_time_ok = (array[0] >= 0) ? array[0] : 0;
1660
1661 e = field_start(e, BYTES_SENT_FIELD - TIME_FIELD + 1);
1662 val = str2ic(e);
1663 ustat->total_bytes_sent = val;
1664
1665 /* the source might be IPv4 or IPv6, so we always strip the port by
1666 * removing the last colon.
1667 */
1668 b = source_field;
1669 e = field_stop(b + 1);
1670 while (e > b && e[-1] != ':')
1671 e--;
1672 *(char *)(e - 1) = '\0';
1673
1674 /* now instead of copying the src for a simple lookup, we'll link
1675 * to it from the node we're trying to insert. If it returns a
1676 * different value, it was already there. Otherwise we just have
1677 * to dynamically realloc an entry using strdup(). We're using the
1678 * <url> field of the node to store the source address.
1679 */
1680 ustat->node.url.key = (char *)b;
1681 ebpt_old = ebis_insert(&timers[0], &ustat->node.url);
1682
1683 if (ebpt_old != &ustat->node.url) {
1684 struct url_stat *ustat_old;
1685 /* node was already there, let's update previous one */
1686 ustat_old = container_of(ebpt_old, struct url_stat, node.url);
1687 ustat_old->nb_req ++;
1688 ustat_old->nb_err += ustat->nb_err;
1689 ustat_old->total_time += ustat->total_time;
1690 ustat_old->total_time_ok += ustat->total_time_ok;
1691 ustat_old->total_bytes_sent += ustat->total_bytes_sent;
1692 } else {
1693 ustat->url = ustat->node.url.key = strdup(ustat->node.url.key);
1694 ustat = NULL; /* node was used */
1695 }
1696}
1697
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001698void filter_graphs(const char *accept_field, const char *time_field, struct timer **tptr)
1699{
1700 struct timer *t2;
Ryan O'Hara8cb99932017-12-15 10:21:39 -06001701 const char *p;
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001702 int f, err, array[5];
1703
1704 if (!time_field) {
1705 time_field = field_start(accept_field, TIME_FIELD - ACCEPT_FIELD + 1);
1706 if (unlikely(!*time_field)) {
1707 truncated_line(linenum, line);
1708 return;
1709 }
1710 }
1711
Ryan O'Hara8cb99932017-12-15 10:21:39 -06001712 field_stop(time_field + 1);
Willy Tarreaua2b39fb2011-07-10 21:39:35 +02001713 /* we have field TIME_FIELD in [time_field]..[e-1] */
1714
1715 p = time_field;
1716 err = 0;
1717 f = 0;
1718 while (!SEP(*p)) {
1719 array[f] = str2ic(p);
1720 if (array[f] < 0) {
1721 array[f] = -1;
1722 err = 1;
1723 }
1724 if (++f == 5)
1725 break;
1726 SKIP_CHAR(p, '/');
1727 }
1728
1729 if (unlikely(f < 5)) {
1730 parse_err++;
1731 return;
1732 }
1733
1734 /* if we find at least one negative time, we count one error
1735 * with a time equal to the total session time. This will
1736 * emphasize quantum timing effects associated to known
1737 * timeouts. Note that on some buggy machines, it is possible
1738 * that the total time is negative, hence the reason to reset
1739 * it.
1740 */
1741
1742 if (filter & FILT_GRAPH_TIMERS) {
1743 if (err) {
1744 if (array[4] < 0)
1745 array[4] = -1;
1746 t2 = insert_timer(&timers[0], tptr, array[4]); // total time
1747 t2->count++;
1748 } else {
1749 int v;
1750
1751 t2 = insert_timer(&timers[1], tptr, array[0]); t2->count++; // req
1752 t2 = insert_timer(&timers[2], tptr, array[2]); t2->count++; // conn
1753 t2 = insert_timer(&timers[3], tptr, array[3]); t2->count++; // resp
1754
1755 v = array[4] - array[0] - array[1] - array[2] - array[3]; // data time
1756 if (v < 0 && !(filter & FILT_QUIET))
1757 fprintf(stderr, "ERR: %s (%d %d %d %d %d => %d)\n",
1758 line, array[0], array[1], array[2], array[3], array[4], v);
1759 t2 = insert_timer(&timers[4], tptr, v); t2->count++;
1760 lines_out++;
1761 }
1762 } else { /* percentile */
1763 if (err) {
1764 if (array[4] < 0)
1765 array[4] = -1;
1766 t2 = insert_value(&timers[0], tptr, array[4]); // total time
1767 t2->count++;
1768 } else {
1769 int v;
1770
1771 t2 = insert_value(&timers[1], tptr, array[0]); t2->count++; // req
1772 t2 = insert_value(&timers[2], tptr, array[2]); t2->count++; // conn
1773 t2 = insert_value(&timers[3], tptr, array[3]); t2->count++; // resp
1774
1775 v = array[4] - array[0] - array[1] - array[2] - array[3]; // data time
1776 if (v < 0 && !(filter & FILT_QUIET))
1777 fprintf(stderr, "ERR: %s (%d %d %d %d %d => %d)\n",
1778 line, array[0], array[1], array[2], array[3], array[4], v);
1779 t2 = insert_value(&timers[4], tptr, v); t2->count++;
1780 lines_out++;
1781 }
1782 }
1783}
1784
1785
Willy Tarreau72c28532009-01-22 18:56:50 +01001786/*
1787 * Local variables:
1788 * c-indent-level: 8
1789 * c-basic-offset: 8
1790 * End:
1791 */