blob: 388efdf0249af982822d7ad175133a6c880b7917 [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
2 * Health-checks functions.
3 *
Willy Tarreau26c25062009-03-08 09:38:41 +01004 * Copyright 2000-2009 Willy Tarreau <w@1wt.eu>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02005 * Copyright 2007-2009 Krzysztof Piotr Oledzki <ole@ans.pl>
Willy Tarreaubaaee002006-06-26 02:48:02 +02006 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 *
12 */
13
Willy Tarreaub8816082008-01-18 12:18:15 +010014#include <assert.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020015#include <ctype.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020016#include <errno.h>
17#include <fcntl.h>
18#include <stdio.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020019#include <stdlib.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020020#include <string.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020021#include <time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020022#include <unistd.h>
23#include <sys/socket.h>
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +040024#include <sys/types.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020025#include <netinet/in.h>
Willy Tarreau1274bc42009-07-15 07:16:31 +020026#include <netinet/tcp.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020027#include <arpa/inet.h>
28
Willy Tarreau2dd0d472006-06-29 17:53:05 +020029#include <common/compat.h>
30#include <common/config.h>
31#include <common/mini-clist.h>
Willy Tarreau83749182007-04-15 20:56:27 +020032#include <common/standard.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020033#include <common/time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020034
35#include <types/global.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020036
37#include <proto/backend.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020038#include <proto/checks.h>
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +010039#include <proto/buffers.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020040#include <proto/fd.h>
41#include <proto/log.h>
42#include <proto/queue.h>
Willy Tarreauc6f4ce82009-06-10 11:09:37 +020043#include <proto/port_range.h>
Willy Tarreau3d300592007-03-18 18:34:41 +010044#include <proto/proto_http.h>
Willy Tarreaue8c66af2008-01-13 18:40:14 +010045#include <proto/proto_tcp.h>
Willy Tarreau2b5652f2006-12-31 17:46:05 +010046#include <proto/proxy.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020047#include <proto/server.h>
48#include <proto/task.h>
49
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020050const struct check_status check_statuses[HCHK_STATUS_SIZE] = {
51 [HCHK_STATUS_UNKNOWN] = { SRV_CHK_UNKNOWN, "UNK", "Unknown" },
52 [HCHK_STATUS_INI] = { SRV_CHK_UNKNOWN, "INI", "Initializing" },
53 [HCHK_STATUS_START] = { /* SPECIAL STATUS*/ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020054
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010055 [HCHK_STATUS_HANA] = { SRV_CHK_ERROR, "HANA", "Health analyze" },
56
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020057 [HCHK_STATUS_SOCKERR] = { SRV_CHK_ERROR, "SOCKERR", "Socket error" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020058
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020059 [HCHK_STATUS_L4OK] = { SRV_CHK_RUNNING, "L4OK", "Layer4 check passed" },
60 [HCHK_STATUS_L4TOUT] = { SRV_CHK_ERROR, "L4TOUT", "Layer4 timeout" },
61 [HCHK_STATUS_L4CON] = { SRV_CHK_ERROR, "L4CON", "Layer4 connection problem" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020062
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020063 [HCHK_STATUS_L6OK] = { SRV_CHK_RUNNING, "L6OK", "Layer6 check passed" },
64 [HCHK_STATUS_L6TOUT] = { SRV_CHK_ERROR, "L6TOUT", "Layer6 timeout" },
65 [HCHK_STATUS_L6RSP] = { SRV_CHK_ERROR, "L6RSP", "Layer6 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020066
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020067 [HCHK_STATUS_L7TOUT] = { SRV_CHK_ERROR, "L7TOUT", "Layer7 timeout" },
68 [HCHK_STATUS_L7RSP] = { SRV_CHK_ERROR, "L7RSP", "Layer7 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020069
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020070 [HCHK_STATUS_L57DATA] = { /* DUMMY STATUS */ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020071
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020072 [HCHK_STATUS_L7OKD] = { SRV_CHK_RUNNING, "L7OK", "Layer7 check passed" },
73 [HCHK_STATUS_L7OKCD] = { SRV_CHK_RUNNING | SRV_CHK_DISABLE, "L7OKC", "Layer7 check conditionally passed" },
74 [HCHK_STATUS_L7STS] = { SRV_CHK_ERROR, "L7STS", "Layer7 wrong status" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020075};
76
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010077const struct analyze_status analyze_statuses[HANA_STATUS_SIZE] = { /* 0: ignore, 1: error, 2: OK */
78 [HANA_STATUS_UNKNOWN] = { "Unknown", { 0, 0 }},
79
80 [HANA_STATUS_L4_OK] = { "L4 successful connection", { 2, 0 }},
81 [HANA_STATUS_L4_ERR] = { "L4 unsuccessful connection", { 1, 1 }},
82
83 [HANA_STATUS_HTTP_OK] = { "Correct http response", { 0, 2 }},
84 [HANA_STATUS_HTTP_STS] = { "Wrong http response", { 0, 1 }},
85 [HANA_STATUS_HTTP_HDRRSP] = { "Invalid http response (headers)", { 0, 1 }},
86 [HANA_STATUS_HTTP_RSP] = { "Invalid http response", { 0, 1 }},
87
88 [HANA_STATUS_HTTP_READ_ERROR] = { "Read error (http)", { 0, 1 }},
89 [HANA_STATUS_HTTP_READ_TIMEOUT] = { "Read timeout (http)", { 0, 1 }},
90 [HANA_STATUS_HTTP_BROKEN_PIPE] = { "Close from server (http)", { 0, 1 }},
91};
92
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020093/*
94 * Convert check_status code to description
95 */
96const char *get_check_status_description(short check_status) {
97
98 const char *desc;
99
100 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200101 desc = check_statuses[check_status].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200102 else
103 desc = NULL;
104
105 if (desc && *desc)
106 return desc;
107 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200108 return check_statuses[HCHK_STATUS_UNKNOWN].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200109}
110
111/*
112 * Convert check_status code to short info
113 */
114const char *get_check_status_info(short check_status) {
115
116 const char *info;
117
118 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200119 info = check_statuses[check_status].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200120 else
121 info = NULL;
122
123 if (info && *info)
124 return info;
125 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200126 return check_statuses[HCHK_STATUS_UNKNOWN].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200127}
128
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100129const char *get_analyze_status(short analyze_status) {
130
131 const char *desc;
132
133 if (analyze_status < HANA_STATUS_SIZE)
134 desc = analyze_statuses[analyze_status].desc;
135 else
136 desc = NULL;
137
138 if (desc && *desc)
139 return desc;
140 else
141 return analyze_statuses[HANA_STATUS_UNKNOWN].desc;
142}
143
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200144#define SSP_O_VIA 0x0001
145#define SSP_O_HCHK 0x0002
146#define SSP_O_STATUS 0x0004
147
148static void server_status_printf(struct chunk *msg, struct server *s, unsigned options, int xferred) {
149
150 if (options & SSP_O_VIA)
151 chunk_printf(msg, " via %s/%s",
152 s->tracked->proxy->id, s->tracked->id);
153
154
155 if (options & SSP_O_HCHK) {
156 chunk_printf(msg, ", reason: %s", get_check_status_description(s->check_status));
157
158 if (s->check_status >= HCHK_STATUS_L57DATA)
159 chunk_printf(msg, ", code: %d", s->check_code);
160
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200161 if (*s->check_desc) {
162 struct chunk src;
163
164 chunk_printf(msg, ", info: \"");
165
166 chunk_initlen(&src, s->check_desc, 0, strlen(s->check_desc));
167 chunk_asciiencode(msg, &src, '"');
168
169 chunk_printf(msg, "\"");
170 }
171
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100172 if (s->check_duration >= 0)
173 chunk_printf(msg, ", check duration: %ldms", s->check_duration);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200174 }
175
176 if (options & SSP_O_STATUS) {
177 if (xferred)
178 chunk_printf(msg, ". %d active and %d backup servers left.%s"
179 " %d sessions active, %d requeued, %d remaining in queue.\n",
180 s->proxy->srv_act, s->proxy->srv_bck,
181 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
182 s->cur_sess, xferred, s->nbpend);
183 else
184 chunk_printf(msg, ". %d active and %d backup servers online.%s"
185 " %d sessions requeued, %d total in queue.\n",
186 s->proxy->srv_act, s->proxy->srv_bck,
187 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
188 xferred, s->nbpend);
189 }
190}
191
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200192/*
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200193 * Set s->check_status, update s->check_duration and fill s->result with
194 * an adequate SRV_CHK_* value.
195 *
196 * Show information in logs about failed health check if server is UP
197 * or succeeded health checks if server is DOWN.
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200198 */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200199static void set_server_check_status(struct server *s, short status, char *desc) {
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200200
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200201 struct chunk msg;
202
203 if (status == HCHK_STATUS_START) {
204 s->result = SRV_CHK_UNKNOWN; /* no result yet */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200205 s->check_desc[0] = '\0';
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200206 s->check_start = now;
207 return;
208 }
209
210 if (!s->check_status)
211 return;
212
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200213 if (desc && *desc) {
214 strncpy(s->check_desc, desc, HCHK_DESC_LEN-1);
215 s->check_desc[HCHK_DESC_LEN-1] = '\0';
216 } else
217 s->check_desc[0] = '\0';
218
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200219 s->check_status = status;
220 if (check_statuses[status].result)
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100221 s->result = check_statuses[status].result;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200222
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100223 if (status == HCHK_STATUS_HANA)
224 s->check_duration = -1;
225 else if (!tv_iszero(&s->check_start)) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200226 /* set_server_check_status() may be called more than once */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200227 s->check_duration = tv_ms_elapsed(&s->check_start, &now);
228 tv_zero(&s->check_start);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200229 }
230
231 if (s->proxy->options2 & PR_O2_LOGHCHKS &&
232 (((s->health != 0) && (s->result & SRV_CHK_ERROR)) ||
233 ((s->health != s->rise + s->fall - 1) && (s->result & SRV_CHK_RUNNING)) ||
234 ((s->state & SRV_GOINGDOWN) && !(s->result & SRV_CHK_DISABLE)) ||
235 (!(s->state & SRV_GOINGDOWN) && (s->result & SRV_CHK_DISABLE)))) {
236
237 int health, rise, fall, state;
238
239 chunk_init(&msg, trash, sizeof(trash));
240
241 /* FIXME begin: calculate local version of the health/rise/fall/state */
242 health = s->health;
243 rise = s->rise;
244 fall = s->fall;
245 state = s->state;
246
247 if (s->result & SRV_CHK_ERROR) {
248 if (health > rise) {
249 health--; /* still good */
250 } else {
251 if (health == rise)
252 state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
253
254 health = 0;
255 }
256 }
257
258 if (s->result & SRV_CHK_RUNNING) {
259 if (health < rise + fall - 1) {
260 health++; /* was bad, stays for a while */
261
262 if (health == rise)
263 state |= SRV_RUNNING;
264
265 if (health >= rise)
266 health = rise + fall - 1; /* OK now */
267 }
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100268
269 /* clear consecutive_errors if observing is enabled */
270 if (s->onerror)
271 s->consecutive_errors = 0;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200272 }
273 /* FIXME end: calculate local version of the health/rise/fall/state */
274
275 chunk_printf(&msg,
276 "Health check for %sserver %s/%s %s%s",
277 s->state & SRV_BACKUP ? "backup " : "",
278 s->proxy->id, s->id,
279 (s->result & SRV_CHK_DISABLE)?"conditionally ":"",
280 (s->result & SRV_CHK_RUNNING)?"succeeded":"failed");
281
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200282 server_status_printf(&msg, s, SSP_O_HCHK, -1);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200283
284 chunk_printf(&msg, ", status: %d/%d %s.\n",
285 (state & SRV_RUNNING) ? (health - rise + 1) : (health),
286 (state & SRV_RUNNING) ? (fall) : (rise),
287 (state & SRV_RUNNING)?"UP":"DOWN");
288
289 Warning("%s", trash);
290 send_log(s->proxy, LOG_NOTICE, "%s", trash);
291 }
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200292}
293
Willy Tarreau48494c02007-11-30 10:41:39 +0100294/* sends a log message when a backend goes down, and also sets last
295 * change date.
296 */
297static void set_backend_down(struct proxy *be)
298{
299 be->last_change = now.tv_sec;
300 be->down_trans++;
301
302 Alert("%s '%s' has no server available!\n", proxy_type_str(be), be->id);
303 send_log(be, LOG_EMERG, "%s %s has no server available!\n", proxy_type_str(be), be->id);
304}
305
306/* Redistribute pending connections when a server goes down. The number of
307 * connections redistributed is returned.
308 */
309static int redistribute_pending(struct server *s)
310{
311 struct pendconn *pc, *pc_bck, *pc_end;
312 int xferred = 0;
313
314 FOREACH_ITEM_SAFE(pc, pc_bck, &s->pendconns, pc_end, struct pendconn *, list) {
315 struct session *sess = pc->sess;
316 if (sess->be->options & PR_O_REDISP) {
317 /* The REDISP option was specified. We will ignore
318 * cookie and force to balance or use the dispatcher.
319 */
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100320
Krzysztof Piotr Oledzki5a329cf2008-02-22 03:50:19 +0100321 /* it's left to the dispatcher to choose a server */
Willy Tarreau48494c02007-11-30 10:41:39 +0100322 sess->flags &= ~(SN_DIRECT | SN_ASSIGNED | SN_ADDR_SET);
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100323
Willy Tarreau48494c02007-11-30 10:41:39 +0100324 pendconn_free(pc);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200325 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100326 xferred++;
327 }
328 }
329 return xferred;
330}
331
332/* Check for pending connections at the backend, and assign some of them to
333 * the server coming up. The server's weight is checked before being assigned
334 * connections it may not be able to handle. The total number of transferred
335 * connections is returned.
336 */
337static int check_for_pending(struct server *s)
338{
339 int xferred;
340
341 if (!s->eweight)
342 return 0;
343
344 for (xferred = 0; !s->maxconn || xferred < srv_dynamic_maxconn(s); xferred++) {
345 struct session *sess;
346 struct pendconn *p;
347
348 p = pendconn_from_px(s->proxy);
349 if (!p)
350 break;
351 p->sess->srv = s;
352 sess = p->sess;
353 pendconn_free(p);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200354 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100355 }
356 return xferred;
357}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200358
359/* Sets server <s> down, notifies by all available means, recounts the
360 * remaining servers on the proxy and transfers queued sessions whenever
Willy Tarreau5af3a692007-07-24 23:32:33 +0200361 * possible to other servers. It automatically recomputes the number of
362 * servers, but not the map.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200363 */
Willy Tarreau83749182007-04-15 20:56:27 +0200364static void set_server_down(struct server *s)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200365{
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100366 struct server *srv;
367 struct chunk msg;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200368 int xferred;
369
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100370 if (s->health == s->rise || s->tracked) {
Willy Tarreau48494c02007-11-30 10:41:39 +0100371 int srv_was_paused = s->state & SRV_GOINGDOWN;
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200372
373 s->last_change = now.tv_sec;
Willy Tarreau48494c02007-11-30 10:41:39 +0100374 s->state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
Willy Tarreaub625a082007-11-26 01:15:43 +0100375 s->proxy->lbprm.set_server_status_down(s);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200376
377 /* we might have sessions queued on this server and waiting for
378 * a connection. Those which are redispatchable will be queued
379 * to another server or to the proxy itself.
380 */
Willy Tarreau48494c02007-11-30 10:41:39 +0100381 xferred = redistribute_pending(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100382
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200383 chunk_init(&msg, trash, sizeof(trash));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100384
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200385 chunk_printf(&msg,
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100386 "%sServer %s/%s is DOWN", s->state & SRV_BACKUP ? "Backup " : "",
387 s->proxy->id, s->id);
388
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200389 server_status_printf(&msg, s,
390 (s->tracked?SSP_O_VIA:0) | SSP_O_STATUS |
391 ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
392 xferred);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200393
394 Warning("%s", trash);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200395
Willy Tarreau48494c02007-11-30 10:41:39 +0100396 /* we don't send an alert if the server was previously paused */
397 if (srv_was_paused)
398 send_log(s->proxy, LOG_NOTICE, "%s", trash);
399 else
400 send_log(s->proxy, LOG_ALERT, "%s", trash);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200401
Willy Tarreau48494c02007-11-30 10:41:39 +0100402 if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0)
403 set_backend_down(s->proxy);
404
Krzysztof Piotr Oledzki052d4fd2009-10-04 14:52:57 +0200405 s->counters.down_trans++;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100406
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100407 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100408 for(srv = s->tracknext; srv; srv = srv->tracknext)
409 set_server_down(srv);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200410 }
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100411
Willy Tarreaubaaee002006-06-26 02:48:02 +0200412 s->health = 0; /* failure */
413}
414
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100415static void set_server_up(struct server *s) {
416
417 struct server *srv;
418 struct chunk msg;
419 int xferred;
420
421 if (s->health == s->rise || s->tracked) {
422 if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0) {
423 if (s->proxy->last_change < now.tv_sec) // ignore negative times
424 s->proxy->down_time += now.tv_sec - s->proxy->last_change;
425 s->proxy->last_change = now.tv_sec;
426 }
427
428 if (s->last_change < now.tv_sec) // ignore negative times
429 s->down_time += now.tv_sec - s->last_change;
430
431 s->last_change = now.tv_sec;
432 s->state |= SRV_RUNNING;
433
434 if (s->slowstart > 0) {
435 s->state |= SRV_WARMINGUP;
436 if (s->proxy->lbprm.algo & BE_LB_PROP_DYN) {
437 /* For dynamic algorithms, start at the first step of the weight,
438 * without multiplying by BE_WEIGHT_SCALE.
439 */
440 s->eweight = s->uweight;
441 if (s->proxy->lbprm.update_server_eweight)
442 s->proxy->lbprm.update_server_eweight(s);
443 }
444 }
445 s->proxy->lbprm.set_server_status_up(s);
446
447 /* check if we can handle some connections queued at the proxy. We
448 * will take as many as we can handle.
449 */
450 xferred = check_for_pending(s);
451
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200452 chunk_init(&msg, trash, sizeof(trash));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100453
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200454 chunk_printf(&msg,
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100455 "%sServer %s/%s is UP", s->state & SRV_BACKUP ? "Backup " : "",
456 s->proxy->id, s->id);
457
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200458 server_status_printf(&msg, s,
459 (s->tracked?SSP_O_VIA:0) | SSP_O_STATUS |
460 ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
461 -1);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100462
463 Warning("%s", trash);
464 send_log(s->proxy, LOG_NOTICE, "%s", trash);
465
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100466 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100467 for(srv = s->tracknext; srv; srv = srv->tracknext)
468 set_server_up(srv);
469 }
470
471 if (s->health >= s->rise)
472 s->health = s->rise + s->fall - 1; /* OK now */
473
474}
475
476static void set_server_disabled(struct server *s) {
477
478 struct server *srv;
479 struct chunk msg;
480 int xferred;
481
482 s->state |= SRV_GOINGDOWN;
483 s->proxy->lbprm.set_server_status_down(s);
484
485 /* we might have sessions queued on this server and waiting for
486 * a connection. Those which are redispatchable will be queued
487 * to another server or to the proxy itself.
488 */
489 xferred = redistribute_pending(s);
490
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200491 chunk_init(&msg, trash, sizeof(trash));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100492
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200493 chunk_printf(&msg,
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100494 "Load-balancing on %sServer %s/%s is disabled",
495 s->state & SRV_BACKUP ? "Backup " : "",
496 s->proxy->id, s->id);
497
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200498 server_status_printf(&msg, s,
499 (s->tracked?SSP_O_VIA:0) | SSP_O_STATUS |
500 ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
501 -1);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100502
503 Warning("%s", trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100504 send_log(s->proxy, LOG_NOTICE, "%s", trash);
505
506 if (!s->proxy->srv_bck && !s->proxy->srv_act)
507 set_backend_down(s->proxy);
508
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100509 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100510 for(srv = s->tracknext; srv; srv = srv->tracknext)
511 set_server_disabled(srv);
512}
513
514static void set_server_enabled(struct server *s) {
515
516 struct server *srv;
517 struct chunk msg;
518 int xferred;
519
520 s->state &= ~SRV_GOINGDOWN;
521 s->proxy->lbprm.set_server_status_up(s);
522
523 /* check if we can handle some connections queued at the proxy. We
524 * will take as many as we can handle.
525 */
526 xferred = check_for_pending(s);
527
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200528 chunk_init(&msg, trash, sizeof(trash));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100529
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200530 chunk_printf(&msg,
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100531 "Load-balancing on %sServer %s/%s is enabled again",
532 s->state & SRV_BACKUP ? "Backup " : "",
533 s->proxy->id, s->id);
534
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200535 server_status_printf(&msg, s,
536 (s->tracked?SSP_O_VIA:0) | SSP_O_STATUS |
537 ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
538 -1);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100539
540 Warning("%s", trash);
541 send_log(s->proxy, LOG_NOTICE, "%s", trash);
542
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100543 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100544 for(srv = s->tracknext; srv; srv = srv->tracknext)
545 set_server_enabled(srv);
546}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200547
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100548void health_adjust(struct server *s, short status) {
549
550 int failed;
551 int expire;
552
553 /* return now if observing nor health check is not enabled */
554 if (!s->observe || !s->check)
555 return;
556
557 if (s->observe >= HANA_OBS_SIZE)
558 return;
559
560 if (status >= HCHK_STATUS_SIZE || !analyze_statuses[status].desc)
561 return;
562
563 switch (analyze_statuses[status].lr[s->observe - 1]) {
564 case 1:
565 failed = 1;
566 break;
567
568 case 2:
569 failed = 0;
570 break;
571
572 default:
573 return;
574 }
575
576 if (!failed) {
577 /* good: clear consecutive_errors */
578 s->consecutive_errors = 0;
579 return;
580 }
581
582 s->consecutive_errors++;
583
584 if (s->consecutive_errors < s->consecutive_errors_limit)
585 return;
586
587 sprintf(trash, "Detected %d consecutive errors, last one was: %s",
588 s->consecutive_errors, get_analyze_status(status));
589
590 switch (s->onerror) {
591 case HANA_ONERR_FASTINTER:
592 /* force fastinter - nothing to do here as all modes force it */
593 break;
594
595 case HANA_ONERR_SUDDTH:
596 /* simulate a pre-fatal failed health check */
597 if (s->health > s->rise)
598 s->health = s->rise + 1;
599
600 /* no break - fall through */
601
602 case HANA_ONERR_FAILCHK:
603 /* simulate a failed health check */
604 set_server_check_status(s, HCHK_STATUS_HANA, trash);
605
606 if (s->health > s->rise) {
607 s->health--; /* still good */
608 s->counters.failed_checks++;
609 }
610 else
611 set_server_down(s);
612
613 break;
614
615 case HANA_ONERR_MARKDWN:
616 /* mark server down */
617 s->health = s->rise;
618 set_server_check_status(s, HCHK_STATUS_HANA, trash);
619 set_server_down(s);
620
621 break;
622
623 default:
624 /* write a warning? */
625 break;
626 }
627
628 s->consecutive_errors = 0;
629 s->counters.failed_hana++;
630
631 if (s->fastinter) {
632 expire = tick_add(now_ms, MS_TO_TICKS(s->fastinter));
633 if (s->check->expire > expire)
634 s->check->expire = expire;
635 }
636}
637
Willy Tarreaubaaee002006-06-26 02:48:02 +0200638/*
639 * This function is used only for server health-checks. It handles
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200640 * the connection acknowledgement. If the proxy requires L7 health-checks,
641 * it sends the request. In other cases, it calls set_server_check_status()
642 * to set s->check_status, s->check_duration and s->result.
Willy Tarreau83749182007-04-15 20:56:27 +0200643 * The function itself returns 0 if it needs some polling before being called
644 * again, otherwise 1.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200645 */
Willy Tarreau83749182007-04-15 20:56:27 +0200646static int event_srv_chk_w(int fd)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200647{
Willy Tarreau6996e152007-04-30 14:37:43 +0200648 __label__ out_wakeup, out_nowake, out_poll, out_error;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200649 struct task *t = fdtab[fd].owner;
650 struct server *s = t->context;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200651
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +0100652 //fprintf(stderr, "event_srv_chk_w, state=%ld\n", unlikely(fdtab[fd].state));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200653 if (unlikely(fdtab[fd].state == FD_STERROR || (fdtab[fd].ev & FD_POLL_ERR))) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200654 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(errno));
Willy Tarreau6996e152007-04-30 14:37:43 +0200655 goto out_error;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200656 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200657
658 /* here, we know that the connection is established */
Willy Tarreau83749182007-04-15 20:56:27 +0200659
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100660 if (!(s->result & SRV_CHK_ERROR)) {
Willy Tarreaubaaee002006-06-26 02:48:02 +0200661 /* we don't want to mark 'UP' a server on which we detected an error earlier */
Willy Tarreauf3c69202006-07-09 16:42:34 +0200662 if ((s->proxy->options & PR_O_HTTP_CHK) ||
Willy Tarreau23677902007-05-08 23:50:35 +0200663 (s->proxy->options & PR_O_SSL3_CHK) ||
664 (s->proxy->options & PR_O_SMTP_CHK)) {
Willy Tarreaubaaee002006-06-26 02:48:02 +0200665 int ret;
Willy Tarreauf3c69202006-07-09 16:42:34 +0200666 /* we want to check if this host replies to HTTP or SSLv3 requests
Willy Tarreaubaaee002006-06-26 02:48:02 +0200667 * so we'll send the request, and won't wake the checker up now.
668 */
Willy Tarreauf3c69202006-07-09 16:42:34 +0200669
670 if (s->proxy->options & PR_O_SSL3_CHK) {
671 /* SSL requires that we put Unix time in the request */
Willy Tarreaub7f694f2008-06-22 17:18:02 +0200672 int gmt_time = htonl(date.tv_sec);
Willy Tarreauf3c69202006-07-09 16:42:34 +0200673 memcpy(s->proxy->check_req + 11, &gmt_time, 4);
674 }
675
Willy Tarreaubaaee002006-06-26 02:48:02 +0200676 ret = send(fd, s->proxy->check_req, s->proxy->check_len, MSG_DONTWAIT | MSG_NOSIGNAL);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200677 if (ret == s->proxy->check_len) {
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +0100678 /* we allow up to <timeout.check> if nonzero for a responce */
Willy Tarreau7cd9d942008-12-21 13:00:41 +0100679 if (s->proxy->timeout.check)
680 t->expire = tick_add_ifset(now_ms, s->proxy->timeout.check);
Willy Tarreauf161a342007-04-08 16:59:42 +0200681 EV_FD_SET(fd, DIR_RD); /* prepare for reading reply */
Willy Tarreau83749182007-04-15 20:56:27 +0200682 goto out_nowake;
683 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200684 else if (ret == 0 || errno == EAGAIN)
685 goto out_poll;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200686 else {
687 switch (errno) {
688 case ECONNREFUSED:
689 case ENETUNREACH:
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200690 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(errno));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200691 break;
692
693 default:
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200694 set_server_check_status(s, HCHK_STATUS_SOCKERR, strerror(errno));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200695 }
696
Willy Tarreau6996e152007-04-30 14:37:43 +0200697 goto out_error;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200698 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200699 }
700 else {
Willy Tarreau6996e152007-04-30 14:37:43 +0200701 /* We have no data to send to check the connection, and
702 * getsockopt() will not inform us whether the connection
703 * is still pending. So we'll reuse connect() to check the
704 * state of the socket. This has the advantage of givig us
705 * the following info :
706 * - error
707 * - connecting (EALREADY, EINPROGRESS)
708 * - connected (EISCONN, 0)
709 */
710
711 struct sockaddr_in sa;
712
713 sa = (s->check_addr.sin_addr.s_addr) ? s->check_addr : s->addr;
714 sa.sin_port = htons(s->check_port);
715
716 if (connect(fd, (struct sockaddr *)&sa, sizeof(sa)) == 0)
717 errno = 0;
718
719 if (errno == EALREADY || errno == EINPROGRESS)
720 goto out_poll;
721
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200722 if (errno && errno != EISCONN) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200723 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(errno));
Willy Tarreau6996e152007-04-30 14:37:43 +0200724 goto out_error;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200725 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200726
Willy Tarreaubaaee002006-06-26 02:48:02 +0200727 /* good TCP connection is enough */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200728 set_server_check_status(s, HCHK_STATUS_L4OK, NULL);
Willy Tarreau6996e152007-04-30 14:37:43 +0200729 goto out_wakeup;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200730 }
731 }
Willy Tarreau83749182007-04-15 20:56:27 +0200732 out_wakeup:
Willy Tarreaufdccded2008-08-29 18:19:04 +0200733 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreau83749182007-04-15 20:56:27 +0200734 out_nowake:
735 EV_FD_CLR(fd, DIR_WR); /* nothing more to write */
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100736 fdtab[fd].ev &= ~FD_POLL_OUT;
Willy Tarreau83749182007-04-15 20:56:27 +0200737 return 1;
Willy Tarreau6996e152007-04-30 14:37:43 +0200738 out_poll:
739 /* The connection is still pending. We'll have to poll it
740 * before attempting to go further. */
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100741 fdtab[fd].ev &= ~FD_POLL_OUT;
Willy Tarreau6996e152007-04-30 14:37:43 +0200742 return 0;
743 out_error:
Willy Tarreau6996e152007-04-30 14:37:43 +0200744 fdtab[fd].state = FD_STERROR;
745 goto out_wakeup;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200746}
747
748
749/*
Willy Tarreauf3c69202006-07-09 16:42:34 +0200750 * This function is used only for server health-checks. It handles the server's
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200751 * reply to an HTTP request or SSL HELLO. It calls set_server_check_status() to
752 * update s->check_status, s->check_duration and s->result.
753
754 * The set_server_check_status function is called with HCHK_STATUS_L7OKD if
755 * an HTTP server replies HTTP 2xx or 3xx (valid responses), if an SMTP server
756 * returns 2xx, HCHK_STATUS_L6OK if an SSL server returns at least 5 bytes in
757 * response to an SSL HELLO (the principle is that this is enough to
758 * distinguish between an SSL server and a pure TCP relay). All other cases will
759 * call it with a proper error status like HCHK_STATUS_L7STS, HCHK_STATUS_L6RSP,
760 * etc.
761 *
762 * The function returns 0 if it needs to be called again after some polling,
763 * otherwise non-zero..
Willy Tarreaubaaee002006-06-26 02:48:02 +0200764 */
Willy Tarreau83749182007-04-15 20:56:27 +0200765static int event_srv_chk_r(int fd)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200766{
Willy Tarreau83749182007-04-15 20:56:27 +0200767 __label__ out_wakeup;
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100768 int len;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200769 struct task *t = fdtab[fd].owner;
770 struct server *s = t->context;
771 int skerr;
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200772 char *desc;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200773 socklen_t lskerr = sizeof(skerr);
774
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100775 len = -1;
Willy Tarreau83749182007-04-15 20:56:27 +0200776
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100777 if (unlikely((s->result & SRV_CHK_ERROR) ||
778 (fdtab[fd].state == FD_STERROR) ||
Willy Tarreau83749182007-04-15 20:56:27 +0200779 (fdtab[fd].ev & FD_POLL_ERR) ||
780 (getsockopt(fd, SOL_SOCKET, SO_ERROR, &skerr, &lskerr) == -1) ||
781 (skerr != 0))) {
782 /* in case of TCP only, this tells us if the connection failed */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200783
784 if (!(s->result & SRV_CHK_ERROR))
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200785 set_server_check_status(s, HCHK_STATUS_SOCKERR, NULL);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200786
Willy Tarreau83749182007-04-15 20:56:27 +0200787 goto out_wakeup;
788 }
789
Willy Tarreau83749182007-04-15 20:56:27 +0200790 /* Warning! Linux returns EAGAIN on SO_ERROR if data are still available
791 * but the connection was closed on the remote end. Fortunately, recv still
792 * works correctly and we don't need to do the getsockopt() on linux.
793 */
Krzysztof Oledzki6b3f8b42007-10-11 18:41:08 +0200794 len = recv(fd, trash, sizeof(trash), MSG_NOSIGNAL);
Willy Tarreau83749182007-04-15 20:56:27 +0200795 if (unlikely(len < 0 && errno == EAGAIN)) {
796 /* we want some polling to happen first */
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100797 fdtab[fd].ev &= ~FD_POLL_IN;
Willy Tarreau83749182007-04-15 20:56:27 +0200798 return 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200799 }
800
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200801 if (len < sizeof(trash))
802 trash[len] = '\0';
803 else
804 trash[len-1] = '\0';
805
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100806 /* Note: the response will only be accepted if read at once */
807 if (s->proxy->options & PR_O_HTTP_CHK) {
808 /* Check if the server speaks HTTP 1.X */
809 if ((len < strlen("HTTP/1.0 000\r")) ||
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200810 (memcmp(trash, "HTTP/1.", 7) != 0 ||
811 (trash[12] != ' ' && trash[12] != '\r')) ||
812 !isdigit(trash[9]) || !isdigit(trash[10]) || !isdigit(trash[11])) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200813
814 cut_crlf(trash);
815 set_server_check_status(s, HCHK_STATUS_L7RSP, trash);
816
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100817 goto out_wakeup;
818 }
819
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200820 s->check_code = str2uic(&trash[9]);
821
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200822 desc = ltrim(&trash[12], ' ');
823 cut_crlf(desc);
824
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100825 /* check the reply : HTTP/1.X 2xx and 3xx are OK */
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200826 if (trash[9] == '2' || trash[9] == '3')
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200827 set_server_check_status(s, HCHK_STATUS_L7OKD, desc);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200828 else if ((s->proxy->options & PR_O_DISABLE404) &&
Willy Tarreau48494c02007-11-30 10:41:39 +0100829 (s->state & SRV_RUNNING) &&
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200830 (s->check_code == 404))
831 /* 404 may be accepted as "stopping" only if the server was up */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200832 set_server_check_status(s, HCHK_STATUS_L7OKCD, desc);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200833 else
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200834 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100835 }
836 else if (s->proxy->options & PR_O_SSL3_CHK) {
837 /* Check for SSLv3 alert or handshake */
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200838 if ((len >= 5) && (trash[0] == 0x15 || trash[0] == 0x16))
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200839 set_server_check_status(s, HCHK_STATUS_L6OK, NULL);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200840 else
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200841 set_server_check_status(s, HCHK_STATUS_L6RSP, NULL);
Willy Tarreau6996e152007-04-30 14:37:43 +0200842 }
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100843 else if (s->proxy->options & PR_O_SMTP_CHK) {
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200844 /* Check if the server speaks SMTP */
845 if ((len < strlen("000\r")) ||
846 (trash[3] != ' ' && trash[3] != '\r') ||
847 !isdigit(trash[0]) || !isdigit(trash[1]) || !isdigit(trash[2])) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200848
849 cut_crlf(trash);
850 set_server_check_status(s, HCHK_STATUS_L7RSP, trash);
851
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200852 goto out_wakeup;
853 }
854
855 s->check_code = str2uic(&trash[0]);
856
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200857 desc = ltrim(&trash[3], ' ');
858 cut_crlf(desc);
859
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100860 /* Check for SMTP code 2xx (should be 250) */
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200861 if (trash[0] == '2')
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200862 set_server_check_status(s, HCHK_STATUS_L7OKD, desc);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200863 else
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200864 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
Willy Tarreau6996e152007-04-30 14:37:43 +0200865 }
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100866 else {
867 /* other checks are valid if the connection succeeded anyway */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200868 set_server_check_status(s, HCHK_STATUS_L4OK, NULL);
Willy Tarreau23677902007-05-08 23:50:35 +0200869 }
Willy Tarreau83749182007-04-15 20:56:27 +0200870
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100871 out_wakeup:
872 if (s->result & SRV_CHK_ERROR)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200873 fdtab[fd].state = FD_STERROR;
874
Willy Tarreauf161a342007-04-08 16:59:42 +0200875 EV_FD_CLR(fd, DIR_RD);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200876 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100877 fdtab[fd].ev &= ~FD_POLL_IN;
Willy Tarreau83749182007-04-15 20:56:27 +0200878 return 1;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200879}
880
881/*
882 * manages a server health-check. Returns
883 * the time the task accepts to wait, or TIME_ETERNITY for infinity.
884 */
Willy Tarreau26c25062009-03-08 09:38:41 +0100885struct task *process_chk(struct task *t)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200886{
Willy Tarreaue3838802009-03-21 18:58:32 +0100887 int attempts = 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200888 struct server *s = t->context;
889 struct sockaddr_in sa;
890 int fd;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +0200891 int rv;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200892
893 //fprintf(stderr, "process_chk: task=%p\n", t);
894
895 new_chk:
Willy Tarreaue3838802009-03-21 18:58:32 +0100896 if (attempts++ > 0) {
897 /* we always fail to create a server, let's stop insisting... */
898 while (tick_is_expired(t->expire, now_ms))
899 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
900 return t;
901 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200902 fd = s->curfd;
903 if (fd < 0) { /* no check currently running */
904 //fprintf(stderr, "process_chk: 2\n");
Willy Tarreau26c25062009-03-08 09:38:41 +0100905 if (!tick_is_expired(t->expire, now_ms)) /* woke up too early */
906 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200907
908 /* we don't send any health-checks when the proxy is stopped or when
909 * the server should not be checked.
910 */
911 if (!(s->state & SRV_CHECKED) || s->proxy->state == PR_STSTOPPED) {
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200912 while (tick_is_expired(t->expire, now_ms))
913 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
Willy Tarreau26c25062009-03-08 09:38:41 +0100914 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200915 }
916
917 /* we'll initiate a new check */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200918 set_server_check_status(s, HCHK_STATUS_START, NULL);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200919 if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) != -1) {
920 if ((fd < global.maxsock) &&
921 (fcntl(fd, F_SETFL, O_NONBLOCK) != -1) &&
922 (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char *) &one, sizeof(one)) != -1)) {
923 //fprintf(stderr, "process_chk: 3\n");
924
Willy Tarreau9edd1612007-10-18 18:07:48 +0200925 if (s->proxy->options & PR_O_TCP_NOLING) {
926 /* We don't want to useless data */
927 setsockopt(fd, SOL_SOCKET, SO_LINGER, (struct linger *) &nolinger, sizeof(struct linger));
928 }
Willy Tarreau2ea3abb2007-03-25 16:45:16 +0200929
Willy Tarreau0f03c6f2007-03-25 20:46:19 +0200930 if (s->check_addr.sin_addr.s_addr)
931 /* we'll connect to the check addr specified on the server */
Willy Tarreau2ea3abb2007-03-25 16:45:16 +0200932 sa = s->check_addr;
Willy Tarreau2ea3abb2007-03-25 16:45:16 +0200933 else
Willy Tarreau0f03c6f2007-03-25 20:46:19 +0200934 /* we'll connect to the addr on the server */
Willy Tarreau2ea3abb2007-03-25 16:45:16 +0200935 sa = s->addr;
Willy Tarreau0f03c6f2007-03-25 20:46:19 +0200936
Willy Tarreaubaaee002006-06-26 02:48:02 +0200937 /* we'll connect to the check port on the server */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200938 sa.sin_port = htons(s->check_port);
939
940 /* allow specific binding :
941 * - server-specific at first
942 * - proxy-specific next
943 */
944 if (s->state & SRV_BIND_SRC) {
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100945 struct sockaddr_in *remote = NULL;
946 int ret, flags = 0;
Willy Tarreau163c5322006-11-14 16:18:41 +0100947
Willy Tarreaucf1d5722008-02-14 20:28:18 +0100948#if defined(CONFIG_HAP_CTTPROXY) || defined(CONFIG_HAP_LINUX_TPROXY)
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100949 if ((s->state & SRV_TPROXY_MASK) == SRV_TPROXY_ADDR) {
950 remote = (struct sockaddr_in *)&s->tproxy_addr;
951 flags = 3;
952 }
Willy Tarreaucf1d5722008-02-14 20:28:18 +0100953#endif
Willy Tarreauc76721d2009-02-04 20:20:58 +0100954#ifdef SO_BINDTODEVICE
955 /* Note: this might fail if not CAP_NET_RAW */
956 if (s->iface_name)
957 setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE,
Willy Tarreau604e8302009-03-06 00:48:23 +0100958 s->iface_name, s->iface_len + 1);
Willy Tarreauc76721d2009-02-04 20:20:58 +0100959#endif
Willy Tarreauc6f4ce82009-06-10 11:09:37 +0200960 if (s->sport_range) {
961 int bind_attempts = 10; /* should be more than enough to find a spare port */
962 struct sockaddr_in src;
963
964 ret = 1;
965 src = s->source_addr;
966
967 do {
968 /* note: in case of retry, we may have to release a previously
969 * allocated port, hence this loop's construct.
970 */
Willy Tarreau8d5d77e2009-10-18 07:25:52 +0200971 port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
972 fdinfo[fd].port_range = NULL;
Willy Tarreauc6f4ce82009-06-10 11:09:37 +0200973
974 if (!bind_attempts)
975 break;
976 bind_attempts--;
977
Willy Tarreau8d5d77e2009-10-18 07:25:52 +0200978 fdinfo[fd].local_port = port_range_alloc_port(s->sport_range);
979 if (!fdinfo[fd].local_port)
Willy Tarreauc6f4ce82009-06-10 11:09:37 +0200980 break;
981
Willy Tarreau8d5d77e2009-10-18 07:25:52 +0200982 fdinfo[fd].port_range = s->sport_range;
983 src.sin_port = htons(fdinfo[fd].local_port);
Willy Tarreauc6f4ce82009-06-10 11:09:37 +0200984
985 ret = tcpv4_bind_socket(fd, flags, &src, remote);
986 } while (ret != 0); /* binding NOK */
987 }
988 else {
989 ret = tcpv4_bind_socket(fd, flags, &s->source_addr, remote);
990 }
991
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100992 if (ret) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200993 set_server_check_status(s, HCHK_STATUS_SOCKERR, NULL);
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100994 switch (ret) {
995 case 1:
996 Alert("Cannot bind to source address before connect() for server %s/%s. Aborting.\n",
997 s->proxy->id, s->id);
998 break;
999 case 2:
Willy Tarreau163c5322006-11-14 16:18:41 +01001000 Alert("Cannot bind to tproxy source address before connect() for server %s/%s. Aborting.\n",
1001 s->proxy->id, s->id);
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001002 break;
Willy Tarreau163c5322006-11-14 16:18:41 +01001003 }
1004 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001005 }
1006 else if (s->proxy->options & PR_O_BIND_SRC) {
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001007 struct sockaddr_in *remote = NULL;
1008 int ret, flags = 0;
1009
Willy Tarreaucf1d5722008-02-14 20:28:18 +01001010#if defined(CONFIG_HAP_CTTPROXY) || defined(CONFIG_HAP_LINUX_TPROXY)
Willy Tarreau163c5322006-11-14 16:18:41 +01001011 if ((s->proxy->options & PR_O_TPXY_MASK) == PR_O_TPXY_ADDR) {
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001012 remote = (struct sockaddr_in *)&s->proxy->tproxy_addr;
1013 flags = 3;
1014 }
Willy Tarreaucf1d5722008-02-14 20:28:18 +01001015#endif
Willy Tarreaud53f96b2009-02-04 18:46:54 +01001016#ifdef SO_BINDTODEVICE
1017 /* Note: this might fail if not CAP_NET_RAW */
1018 if (s->proxy->iface_name)
1019 setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE,
Willy Tarreau604e8302009-03-06 00:48:23 +01001020 s->proxy->iface_name, s->proxy->iface_len + 1);
Willy Tarreaud53f96b2009-02-04 18:46:54 +01001021#endif
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001022 ret = tcpv4_bind_socket(fd, flags, &s->proxy->source_addr, remote);
1023 if (ret) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001024 set_server_check_status(s, HCHK_STATUS_SOCKERR, NULL);
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001025 switch (ret) {
1026 case 1:
1027 Alert("Cannot bind to source address before connect() for %s '%s'. Aborting.\n",
1028 proxy_type_str(s->proxy), s->proxy->id);
1029 break;
1030 case 2:
Willy Tarreau2b5652f2006-12-31 17:46:05 +01001031 Alert("Cannot bind to tproxy source address before connect() for %s '%s'. Aborting.\n",
1032 proxy_type_str(s->proxy), s->proxy->id);
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001033 break;
Willy Tarreau163c5322006-11-14 16:18:41 +01001034 }
1035 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001036 }
1037
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001038 if (s->result == SRV_CHK_UNKNOWN) {
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +04001039#if defined(TCP_QUICKACK)
Willy Tarreau1274bc42009-07-15 07:16:31 +02001040 /* disabling tcp quick ack now allows
1041 * the request to leave the machine with
1042 * the first ACK.
1043 */
1044 if (s->proxy->options2 & PR_O2_SMARTCON)
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +04001045 setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, (char *) &zero, sizeof(zero));
Willy Tarreau1274bc42009-07-15 07:16:31 +02001046#endif
Willy Tarreaubaaee002006-06-26 02:48:02 +02001047 if ((connect(fd, (struct sockaddr *)&sa, sizeof(sa)) != -1) || (errno == EINPROGRESS)) {
1048 /* OK, connection in progress or established */
1049
1050 //fprintf(stderr, "process_chk: 4\n");
1051
1052 s->curfd = fd; /* that's how we know a test is in progress ;-) */
Willy Tarreau7a966482007-04-15 10:58:02 +02001053 fd_insert(fd);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001054 fdtab[fd].owner = t;
Willy Tarreau54469402006-07-29 16:59:06 +02001055 fdtab[fd].cb[DIR_RD].f = &event_srv_chk_r;
1056 fdtab[fd].cb[DIR_RD].b = NULL;
1057 fdtab[fd].cb[DIR_WR].f = &event_srv_chk_w;
1058 fdtab[fd].cb[DIR_WR].b = NULL;
Willy Tarreau8d5d77e2009-10-18 07:25:52 +02001059 fdinfo[fd].peeraddr = (struct sockaddr *)&sa;
1060 fdinfo[fd].peerlen = sizeof(sa);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001061 fdtab[fd].state = FD_STCONN; /* connection in progress */
Willy Tarreaufb14edc2009-06-14 15:24:37 +02001062 fdtab[fd].flags = FD_FL_TCP | FD_FL_TCP_NODELAY;
Willy Tarreauf161a342007-04-08 16:59:42 +02001063 EV_FD_SET(fd, DIR_WR); /* for connect status */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001064#ifdef DEBUG_FULL
Willy Tarreauf161a342007-04-08 16:59:42 +02001065 assert (!EV_FD_ISSET(fd, DIR_RD));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001066#endif
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001067 //fprintf(stderr, "process_chk: 4+, %lu\n", __tv_to_ms(&s->proxy->timeout.connect));
1068 /* we allow up to min(inter, timeout.connect) for a connection
1069 * to establish but only when timeout.check is set
1070 * as it may be to short for a full check otherwise
1071 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001072 t->expire = tick_add(now_ms, MS_TO_TICKS(s->inter));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001073
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001074 if (s->proxy->timeout.check && s->proxy->timeout.connect) {
1075 int t_con = tick_add(now_ms, s->proxy->timeout.connect);
1076 t->expire = tick_first(t->expire, t_con);
Willy Tarreau60548192008-02-17 11:34:10 +01001077 }
Willy Tarreau26c25062009-03-08 09:38:41 +01001078 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001079 }
1080 else if (errno != EALREADY && errno != EISCONN && errno != EAGAIN) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +02001081 /* a real error */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001082
1083 switch (errno) {
1084 /* FIXME: is it possible to get ECONNREFUSED/ENETUNREACH with O_NONBLOCK? */
1085 case ECONNREFUSED:
1086 case ENETUNREACH:
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001087 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(errno));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001088 break;
1089
1090 default:
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001091 set_server_check_status(s, HCHK_STATUS_SOCKERR, strerror(errno));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001092 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001093 }
1094 }
1095 }
Willy Tarreau8d5d77e2009-10-18 07:25:52 +02001096 port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
1097 fdinfo[fd].port_range = NULL;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001098 close(fd); /* socket creation error */
1099 }
1100
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001101 if (s->result == SRV_CHK_UNKNOWN) { /* nothing done */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001102 //fprintf(stderr, "process_chk: 6\n");
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001103 while (tick_is_expired(t->expire, now_ms))
1104 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001105 goto new_chk; /* may be we should initialize a new check */
1106 }
1107
1108 /* here, we have seen a failure */
1109 if (s->health > s->rise) {
1110 s->health--; /* still good */
Krzysztof Piotr Oledzki052d4fd2009-10-04 14:52:57 +02001111 s->counters.failed_checks++;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001112 }
1113 else
1114 set_server_down(s);
1115
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001116 //fprintf(stderr, "process_chk: 7, %lu\n", __tv_to_ms(&s->proxy->timeout.connect));
1117 /* we allow up to min(inter, timeout.connect) for a connection
1118 * to establish but only when timeout.check is set
1119 * as it may be to short for a full check otherwise
1120 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001121 while (tick_is_expired(t->expire, now_ms)) {
1122 int t_con;
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001123
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001124 t_con = tick_add(t->expire, s->proxy->timeout.connect);
1125 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001126
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001127 if (s->proxy->timeout.check)
1128 t->expire = tick_first(t->expire, t_con);
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001129 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001130 goto new_chk;
1131 }
1132 else {
1133 //fprintf(stderr, "process_chk: 8\n");
1134 /* there was a test running */
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001135 if ((s->result & (SRV_CHK_ERROR|SRV_CHK_RUNNING)) == SRV_CHK_RUNNING) { /* good server detected */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001136 //fprintf(stderr, "process_chk: 9\n");
Krzysztof Oledzki85130942007-10-22 16:21:10 +02001137
Willy Tarreau9909fc12007-11-30 17:42:05 +01001138 if (s->state & SRV_WARMINGUP) {
1139 if (now.tv_sec < s->last_change || now.tv_sec >= s->last_change + s->slowstart) {
1140 s->state &= ~SRV_WARMINGUP;
1141 if (s->proxy->lbprm.algo & BE_LB_PROP_DYN)
1142 s->eweight = s->uweight * BE_WEIGHT_SCALE;
1143 if (s->proxy->lbprm.update_server_eweight)
1144 s->proxy->lbprm.update_server_eweight(s);
1145 }
1146 else if (s->proxy->lbprm.algo & BE_LB_PROP_DYN) {
1147 /* for dynamic algorithms, let's update the weight */
Willy Tarreau5542af62007-12-03 02:04:00 +01001148 s->eweight = (BE_WEIGHT_SCALE * (now.tv_sec - s->last_change) +
1149 s->slowstart - 1) / s->slowstart;
Willy Tarreau9909fc12007-11-30 17:42:05 +01001150 s->eweight *= s->uweight;
1151 if (s->proxy->lbprm.update_server_eweight)
1152 s->proxy->lbprm.update_server_eweight(s);
1153 }
1154 /* probably that we can refill this server with a bit more connections */
1155 check_for_pending(s);
1156 }
1157
Willy Tarreau48494c02007-11-30 10:41:39 +01001158 /* we may have to add/remove this server from the LB group */
1159 if ((s->state & SRV_RUNNING) && (s->proxy->options & PR_O_DISABLE404)) {
1160 if ((s->state & SRV_GOINGDOWN) &&
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +01001161 ((s->result & (SRV_CHK_RUNNING|SRV_CHK_DISABLE)) == SRV_CHK_RUNNING))
1162 set_server_enabled(s);
Willy Tarreau48494c02007-11-30 10:41:39 +01001163 else if (!(s->state & SRV_GOINGDOWN) &&
1164 ((s->result & (SRV_CHK_RUNNING | SRV_CHK_DISABLE)) ==
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +01001165 (SRV_CHK_RUNNING | SRV_CHK_DISABLE)))
1166 set_server_disabled(s);
Willy Tarreau48494c02007-11-30 10:41:39 +01001167 }
1168
Krzysztof Oledzki85130942007-10-22 16:21:10 +02001169 if (s->health < s->rise + s->fall - 1) {
1170 s->health++; /* was bad, stays for a while */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001171
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +01001172 set_server_up(s);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001173 }
1174 s->curfd = -1; /* no check running anymore */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001175 fd_delete(fd);
Willy Tarreau44ec0f02007-10-14 23:47:04 +02001176
1177 rv = 0;
1178 if (global.spread_checks > 0) {
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001179 rv = srv_getinter(s) * global.spread_checks / 100;
Willy Tarreau44ec0f02007-10-14 23:47:04 +02001180 rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0)));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001181 //fprintf(stderr, "process_chk(%p): (%d+/-%d%%) random=%d\n", s, srv_getinter(s), global.spread_checks, rv);
Willy Tarreau44ec0f02007-10-14 23:47:04 +02001182 }
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001183 t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(s) + rv));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001184 goto new_chk;
1185 }
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001186 else if ((s->result & SRV_CHK_ERROR) || tick_is_expired(t->expire, now_ms)) {
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001187 if (!(s->result & SRV_CHK_ERROR)) {
1188 if (!EV_FD_ISSET(fd, DIR_RD)) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001189 set_server_check_status(s, HCHK_STATUS_L4TOUT, NULL);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001190 } else {
1191 if (s->proxy->options & PR_O_SSL3_CHK)
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001192 set_server_check_status(s, HCHK_STATUS_L6TOUT, NULL);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001193 else /* HTTP, SMTP */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001194 set_server_check_status(s, HCHK_STATUS_L7TOUT, NULL);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001195 }
1196 }
1197
Willy Tarreaubaaee002006-06-26 02:48:02 +02001198 //fprintf(stderr, "process_chk: 10\n");
1199 /* failure or timeout detected */
1200 if (s->health > s->rise) {
1201 s->health--; /* still good */
Krzysztof Piotr Oledzki052d4fd2009-10-04 14:52:57 +02001202 s->counters.failed_checks++;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001203 }
1204 else
1205 set_server_down(s);
1206 s->curfd = -1;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001207 fd_delete(fd);
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001208
1209 rv = 0;
1210 if (global.spread_checks > 0) {
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001211 rv = srv_getinter(s) * global.spread_checks / 100;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001212 rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0)));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001213 //fprintf(stderr, "process_chk(%p): (%d+/-%d%%) random=%d\n", s, srv_getinter(s), global.spread_checks, rv);
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001214 }
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001215 t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(s) + rv));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001216 goto new_chk;
1217 }
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001218 /* if result is unknown and there's no timeout, we have to wait again */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001219 }
1220 //fprintf(stderr, "process_chk: 11\n");
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001221 s->result = SRV_CHK_UNKNOWN;
Willy Tarreau26c25062009-03-08 09:38:41 +01001222 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001223}
1224
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001225/*
1226 * Start health-check.
1227 * Returns 0 if OK, -1 if error, and prints the error in this case.
1228 */
1229int start_checks() {
1230
1231 struct proxy *px;
1232 struct server *s;
1233 struct task *t;
1234 int nbchk=0, mininter=0, srvpos=0;
1235
Willy Tarreau2c43a1e2007-10-14 23:05:39 +02001236 /* 1- count the checkers to run simultaneously.
1237 * We also determine the minimum interval among all of those which
1238 * have an interval larger than SRV_CHK_INTER_THRES. This interval
1239 * will be used to spread their start-up date. Those which have
1240 * a shorter interval will start independantly and will not dictate
1241 * too short an interval for all others.
1242 */
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001243 for (px = proxy; px; px = px->next) {
1244 for (s = px->srv; s; s = s->next) {
1245 if (!(s->state & SRV_CHECKED))
1246 continue;
1247
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001248 if ((srv_getinter(s) >= SRV_CHK_INTER_THRES) &&
1249 (!mininter || mininter > srv_getinter(s)))
1250 mininter = srv_getinter(s);
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001251
1252 nbchk++;
1253 }
1254 }
1255
1256 if (!nbchk)
1257 return 0;
1258
1259 srand((unsigned)time(NULL));
1260
1261 /*
1262 * 2- start them as far as possible from each others. For this, we will
1263 * start them after their interval set to the min interval divided by
1264 * the number of servers, weighted by the server's position in the list.
1265 */
1266 for (px = proxy; px; px = px->next) {
1267 for (s = px->srv; s; s = s->next) {
1268 if (!(s->state & SRV_CHECKED))
1269 continue;
1270
Willy Tarreaua4613182009-03-21 18:13:21 +01001271 if ((t = task_new()) == NULL) {
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001272 Alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
1273 return -1;
1274 }
1275
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +02001276 s->check = t;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001277 t->process = process_chk;
1278 t->context = s;
1279
1280 /* check this every ms */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001281 t->expire = tick_add(now_ms,
1282 MS_TO_TICKS(((mininter && mininter >= srv_getinter(s)) ?
1283 mininter : srv_getinter(s)) * srvpos / nbchk));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001284 s->check_start = now;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001285 task_queue(t);
1286
1287 srvpos++;
1288 }
1289 }
1290 return 0;
1291}
Willy Tarreaubaaee002006-06-26 02:48:02 +02001292
1293/*
1294 * Local variables:
1295 * c-indent-level: 8
1296 * c-basic-offset: 8
1297 * End:
1298 */