blob: 723e693eefe68ff43082254df75e4a754531e36d [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
2 * Health-checks functions.
3 *
Willy Tarreau26c25062009-03-08 09:38:41 +01004 * Copyright 2000-2009 Willy Tarreau <w@1wt.eu>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02005 * Copyright 2007-2009 Krzysztof Piotr Oledzki <ole@ans.pl>
Willy Tarreaubaaee002006-06-26 02:48:02 +02006 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 *
12 */
13
Willy Tarreaub8816082008-01-18 12:18:15 +010014#include <assert.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020015#include <ctype.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020016#include <errno.h>
17#include <fcntl.h>
18#include <stdio.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020019#include <stdlib.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020020#include <string.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020021#include <time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020022#include <unistd.h>
23#include <sys/socket.h>
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +040024#include <sys/types.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020025#include <netinet/in.h>
Willy Tarreau1274bc42009-07-15 07:16:31 +020026#include <netinet/tcp.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020027#include <arpa/inet.h>
28
Willy Tarreau2dd0d472006-06-29 17:53:05 +020029#include <common/compat.h>
30#include <common/config.h>
31#include <common/mini-clist.h>
Willy Tarreau83749182007-04-15 20:56:27 +020032#include <common/standard.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020033#include <common/time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020034
35#include <types/global.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020036
37#include <proto/backend.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020038#include <proto/checks.h>
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +010039#include <proto/buffers.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020040#include <proto/fd.h>
41#include <proto/log.h>
42#include <proto/queue.h>
Willy Tarreauc6f4ce82009-06-10 11:09:37 +020043#include <proto/port_range.h>
Willy Tarreau3d300592007-03-18 18:34:41 +010044#include <proto/proto_http.h>
Willy Tarreaue8c66af2008-01-13 18:40:14 +010045#include <proto/proto_tcp.h>
Willy Tarreau2b5652f2006-12-31 17:46:05 +010046#include <proto/proxy.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020047#include <proto/server.h>
48#include <proto/task.h>
49
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020050const struct check_status check_statuses[HCHK_STATUS_SIZE] = {
51 [HCHK_STATUS_UNKNOWN] = { SRV_CHK_UNKNOWN, "UNK", "Unknown" },
52 [HCHK_STATUS_INI] = { SRV_CHK_UNKNOWN, "INI", "Initializing" },
53 [HCHK_STATUS_START] = { /* SPECIAL STATUS*/ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020054
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020055 [HCHK_STATUS_SOCKERR] = { SRV_CHK_ERROR, "SOCKERR", "Socket error" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020056
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020057 [HCHK_STATUS_L4OK] = { SRV_CHK_RUNNING, "L4OK", "Layer4 check passed" },
58 [HCHK_STATUS_L4TOUT] = { SRV_CHK_ERROR, "L4TOUT", "Layer4 timeout" },
59 [HCHK_STATUS_L4CON] = { SRV_CHK_ERROR, "L4CON", "Layer4 connection problem" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020060
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020061 [HCHK_STATUS_L6OK] = { SRV_CHK_RUNNING, "L6OK", "Layer6 check passed" },
62 [HCHK_STATUS_L6TOUT] = { SRV_CHK_ERROR, "L6TOUT", "Layer6 timeout" },
63 [HCHK_STATUS_L6RSP] = { SRV_CHK_ERROR, "L6RSP", "Layer6 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020064
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020065 [HCHK_STATUS_L7TOUT] = { SRV_CHK_ERROR, "L7TOUT", "Layer7 timeout" },
66 [HCHK_STATUS_L7RSP] = { SRV_CHK_ERROR, "L7RSP", "Layer7 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020067
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020068 [HCHK_STATUS_L57DATA] = { /* DUMMY STATUS */ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020069
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020070 [HCHK_STATUS_L7OKD] = { SRV_CHK_RUNNING, "L7OK", "Layer7 check passed" },
71 [HCHK_STATUS_L7OKCD] = { SRV_CHK_RUNNING | SRV_CHK_DISABLE, "L7OKC", "Layer7 check conditionally passed" },
72 [HCHK_STATUS_L7STS] = { SRV_CHK_ERROR, "L7STS", "Layer7 wrong status" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020073};
74
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020075/*
76 * Convert check_status code to description
77 */
78const char *get_check_status_description(short check_status) {
79
80 const char *desc;
81
82 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020083 desc = check_statuses[check_status].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020084 else
85 desc = NULL;
86
87 if (desc && *desc)
88 return desc;
89 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020090 return check_statuses[HCHK_STATUS_UNKNOWN].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020091}
92
93/*
94 * Convert check_status code to short info
95 */
96const char *get_check_status_info(short check_status) {
97
98 const char *info;
99
100 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200101 info = check_statuses[check_status].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200102 else
103 info = NULL;
104
105 if (info && *info)
106 return info;
107 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200108 return check_statuses[HCHK_STATUS_UNKNOWN].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200109}
110
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200111#define SSP_O_VIA 0x0001
112#define SSP_O_HCHK 0x0002
113#define SSP_O_STATUS 0x0004
114
115static void server_status_printf(struct chunk *msg, struct server *s, unsigned options, int xferred) {
116
117 if (options & SSP_O_VIA)
118 chunk_printf(msg, " via %s/%s",
119 s->tracked->proxy->id, s->tracked->id);
120
121
122 if (options & SSP_O_HCHK) {
123 chunk_printf(msg, ", reason: %s", get_check_status_description(s->check_status));
124
125 if (s->check_status >= HCHK_STATUS_L57DATA)
126 chunk_printf(msg, ", code: %d", s->check_code);
127
128 chunk_printf(msg, ", check duration: %lums", s->check_duration);
129 }
130
131 if (options & SSP_O_STATUS) {
132 if (xferred)
133 chunk_printf(msg, ". %d active and %d backup servers left.%s"
134 " %d sessions active, %d requeued, %d remaining in queue.\n",
135 s->proxy->srv_act, s->proxy->srv_bck,
136 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
137 s->cur_sess, xferred, s->nbpend);
138 else
139 chunk_printf(msg, ". %d active and %d backup servers online.%s"
140 " %d sessions requeued, %d total in queue.\n",
141 s->proxy->srv_act, s->proxy->srv_bck,
142 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
143 xferred, s->nbpend);
144 }
145}
146
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200147/*
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200148 * Set s->check_status, update s->check_duration and fill s->result with
149 * an adequate SRV_CHK_* value.
150 *
151 * Show information in logs about failed health check if server is UP
152 * or succeeded health checks if server is DOWN.
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200153 */
154static void set_server_check_status(struct server *s, short status) {
155
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200156 struct chunk msg;
157
158 if (status == HCHK_STATUS_START) {
159 s->result = SRV_CHK_UNKNOWN; /* no result yet */
160 s->check_start = now;
161 return;
162 }
163
164 if (!s->check_status)
165 return;
166
167 s->check_status = status;
168 if (check_statuses[status].result)
169 s->result |= check_statuses[status].result;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200170
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200171 if (!tv_iszero(&s->check_start)) {
172 /* set_server_check_status() may be called more than once */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200173 s->check_duration = tv_ms_elapsed(&s->check_start, &now);
174 tv_zero(&s->check_start);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200175 }
176
177 if (s->proxy->options2 & PR_O2_LOGHCHKS &&
178 (((s->health != 0) && (s->result & SRV_CHK_ERROR)) ||
179 ((s->health != s->rise + s->fall - 1) && (s->result & SRV_CHK_RUNNING)) ||
180 ((s->state & SRV_GOINGDOWN) && !(s->result & SRV_CHK_DISABLE)) ||
181 (!(s->state & SRV_GOINGDOWN) && (s->result & SRV_CHK_DISABLE)))) {
182
183 int health, rise, fall, state;
184
185 chunk_init(&msg, trash, sizeof(trash));
186
187 /* FIXME begin: calculate local version of the health/rise/fall/state */
188 health = s->health;
189 rise = s->rise;
190 fall = s->fall;
191 state = s->state;
192
193 if (s->result & SRV_CHK_ERROR) {
194 if (health > rise) {
195 health--; /* still good */
196 } else {
197 if (health == rise)
198 state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
199
200 health = 0;
201 }
202 }
203
204 if (s->result & SRV_CHK_RUNNING) {
205 if (health < rise + fall - 1) {
206 health++; /* was bad, stays for a while */
207
208 if (health == rise)
209 state |= SRV_RUNNING;
210
211 if (health >= rise)
212 health = rise + fall - 1; /* OK now */
213 }
214 }
215 /* FIXME end: calculate local version of the health/rise/fall/state */
216
217 chunk_printf(&msg,
218 "Health check for %sserver %s/%s %s%s",
219 s->state & SRV_BACKUP ? "backup " : "",
220 s->proxy->id, s->id,
221 (s->result & SRV_CHK_DISABLE)?"conditionally ":"",
222 (s->result & SRV_CHK_RUNNING)?"succeeded":"failed");
223
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200224 server_status_printf(&msg, s, SSP_O_HCHK, -1);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200225
226 chunk_printf(&msg, ", status: %d/%d %s.\n",
227 (state & SRV_RUNNING) ? (health - rise + 1) : (health),
228 (state & SRV_RUNNING) ? (fall) : (rise),
229 (state & SRV_RUNNING)?"UP":"DOWN");
230
231 Warning("%s", trash);
232 send_log(s->proxy, LOG_NOTICE, "%s", trash);
233 }
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200234}
235
Willy Tarreau48494c02007-11-30 10:41:39 +0100236/* sends a log message when a backend goes down, and also sets last
237 * change date.
238 */
239static void set_backend_down(struct proxy *be)
240{
241 be->last_change = now.tv_sec;
242 be->down_trans++;
243
244 Alert("%s '%s' has no server available!\n", proxy_type_str(be), be->id);
245 send_log(be, LOG_EMERG, "%s %s has no server available!\n", proxy_type_str(be), be->id);
246}
247
248/* Redistribute pending connections when a server goes down. The number of
249 * connections redistributed is returned.
250 */
251static int redistribute_pending(struct server *s)
252{
253 struct pendconn *pc, *pc_bck, *pc_end;
254 int xferred = 0;
255
256 FOREACH_ITEM_SAFE(pc, pc_bck, &s->pendconns, pc_end, struct pendconn *, list) {
257 struct session *sess = pc->sess;
258 if (sess->be->options & PR_O_REDISP) {
259 /* The REDISP option was specified. We will ignore
260 * cookie and force to balance or use the dispatcher.
261 */
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100262
Krzysztof Piotr Oledzki5a329cf2008-02-22 03:50:19 +0100263 /* it's left to the dispatcher to choose a server */
Willy Tarreau48494c02007-11-30 10:41:39 +0100264 sess->flags &= ~(SN_DIRECT | SN_ASSIGNED | SN_ADDR_SET);
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100265
Willy Tarreau48494c02007-11-30 10:41:39 +0100266 pendconn_free(pc);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200267 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100268 xferred++;
269 }
270 }
271 return xferred;
272}
273
274/* Check for pending connections at the backend, and assign some of them to
275 * the server coming up. The server's weight is checked before being assigned
276 * connections it may not be able to handle. The total number of transferred
277 * connections is returned.
278 */
279static int check_for_pending(struct server *s)
280{
281 int xferred;
282
283 if (!s->eweight)
284 return 0;
285
286 for (xferred = 0; !s->maxconn || xferred < srv_dynamic_maxconn(s); xferred++) {
287 struct session *sess;
288 struct pendconn *p;
289
290 p = pendconn_from_px(s->proxy);
291 if (!p)
292 break;
293 p->sess->srv = s;
294 sess = p->sess;
295 pendconn_free(p);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200296 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100297 }
298 return xferred;
299}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200300
301/* Sets server <s> down, notifies by all available means, recounts the
302 * remaining servers on the proxy and transfers queued sessions whenever
Willy Tarreau5af3a692007-07-24 23:32:33 +0200303 * possible to other servers. It automatically recomputes the number of
304 * servers, but not the map.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200305 */
Willy Tarreau83749182007-04-15 20:56:27 +0200306static void set_server_down(struct server *s)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200307{
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100308 struct server *srv;
309 struct chunk msg;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200310 int xferred;
311
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100312 if (s->health == s->rise || s->tracked) {
Willy Tarreau48494c02007-11-30 10:41:39 +0100313 int srv_was_paused = s->state & SRV_GOINGDOWN;
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200314
315 s->last_change = now.tv_sec;
Willy Tarreau48494c02007-11-30 10:41:39 +0100316 s->state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
Willy Tarreaub625a082007-11-26 01:15:43 +0100317 s->proxy->lbprm.set_server_status_down(s);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200318
319 /* we might have sessions queued on this server and waiting for
320 * a connection. Those which are redispatchable will be queued
321 * to another server or to the proxy itself.
322 */
Willy Tarreau48494c02007-11-30 10:41:39 +0100323 xferred = redistribute_pending(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100324
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200325 chunk_init(&msg, trash, sizeof(trash));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100326
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200327 chunk_printf(&msg,
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100328 "%sServer %s/%s is DOWN", s->state & SRV_BACKUP ? "Backup " : "",
329 s->proxy->id, s->id);
330
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200331 server_status_printf(&msg, s,
332 (s->tracked?SSP_O_VIA:0) | SSP_O_STATUS |
333 ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
334 xferred);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200335
336 Warning("%s", trash);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200337
Willy Tarreau48494c02007-11-30 10:41:39 +0100338 /* we don't send an alert if the server was previously paused */
339 if (srv_was_paused)
340 send_log(s->proxy, LOG_NOTICE, "%s", trash);
341 else
342 send_log(s->proxy, LOG_ALERT, "%s", trash);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200343
Willy Tarreau48494c02007-11-30 10:41:39 +0100344 if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0)
345 set_backend_down(s->proxy);
346
Willy Tarreaubaaee002006-06-26 02:48:02 +0200347 s->down_trans++;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100348
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100349 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100350 for(srv = s->tracknext; srv; srv = srv->tracknext)
351 set_server_down(srv);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200352 }
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100353
Willy Tarreaubaaee002006-06-26 02:48:02 +0200354 s->health = 0; /* failure */
355}
356
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100357static void set_server_up(struct server *s) {
358
359 struct server *srv;
360 struct chunk msg;
361 int xferred;
362
363 if (s->health == s->rise || s->tracked) {
364 if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0) {
365 if (s->proxy->last_change < now.tv_sec) // ignore negative times
366 s->proxy->down_time += now.tv_sec - s->proxy->last_change;
367 s->proxy->last_change = now.tv_sec;
368 }
369
370 if (s->last_change < now.tv_sec) // ignore negative times
371 s->down_time += now.tv_sec - s->last_change;
372
373 s->last_change = now.tv_sec;
374 s->state |= SRV_RUNNING;
375
376 if (s->slowstart > 0) {
377 s->state |= SRV_WARMINGUP;
378 if (s->proxy->lbprm.algo & BE_LB_PROP_DYN) {
379 /* For dynamic algorithms, start at the first step of the weight,
380 * without multiplying by BE_WEIGHT_SCALE.
381 */
382 s->eweight = s->uweight;
383 if (s->proxy->lbprm.update_server_eweight)
384 s->proxy->lbprm.update_server_eweight(s);
385 }
386 }
387 s->proxy->lbprm.set_server_status_up(s);
388
389 /* check if we can handle some connections queued at the proxy. We
390 * will take as many as we can handle.
391 */
392 xferred = check_for_pending(s);
393
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200394 chunk_init(&msg, trash, sizeof(trash));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100395
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200396 chunk_printf(&msg,
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100397 "%sServer %s/%s is UP", s->state & SRV_BACKUP ? "Backup " : "",
398 s->proxy->id, s->id);
399
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200400 server_status_printf(&msg, s,
401 (s->tracked?SSP_O_VIA:0) | SSP_O_STATUS |
402 ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
403 -1);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100404
405 Warning("%s", trash);
406 send_log(s->proxy, LOG_NOTICE, "%s", trash);
407
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100408 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100409 for(srv = s->tracknext; srv; srv = srv->tracknext)
410 set_server_up(srv);
411 }
412
413 if (s->health >= s->rise)
414 s->health = s->rise + s->fall - 1; /* OK now */
415
416}
417
418static void set_server_disabled(struct server *s) {
419
420 struct server *srv;
421 struct chunk msg;
422 int xferred;
423
424 s->state |= SRV_GOINGDOWN;
425 s->proxy->lbprm.set_server_status_down(s);
426
427 /* we might have sessions queued on this server and waiting for
428 * a connection. Those which are redispatchable will be queued
429 * to another server or to the proxy itself.
430 */
431 xferred = redistribute_pending(s);
432
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200433 chunk_init(&msg, trash, sizeof(trash));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100434
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200435 chunk_printf(&msg,
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100436 "Load-balancing on %sServer %s/%s is disabled",
437 s->state & SRV_BACKUP ? "Backup " : "",
438 s->proxy->id, s->id);
439
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200440 server_status_printf(&msg, s,
441 (s->tracked?SSP_O_VIA:0) | SSP_O_STATUS |
442 ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
443 -1);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100444
445 Warning("%s", trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100446 send_log(s->proxy, LOG_NOTICE, "%s", trash);
447
448 if (!s->proxy->srv_bck && !s->proxy->srv_act)
449 set_backend_down(s->proxy);
450
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100451 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100452 for(srv = s->tracknext; srv; srv = srv->tracknext)
453 set_server_disabled(srv);
454}
455
456static void set_server_enabled(struct server *s) {
457
458 struct server *srv;
459 struct chunk msg;
460 int xferred;
461
462 s->state &= ~SRV_GOINGDOWN;
463 s->proxy->lbprm.set_server_status_up(s);
464
465 /* check if we can handle some connections queued at the proxy. We
466 * will take as many as we can handle.
467 */
468 xferred = check_for_pending(s);
469
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200470 chunk_init(&msg, trash, sizeof(trash));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100471
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200472 chunk_printf(&msg,
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100473 "Load-balancing on %sServer %s/%s is enabled again",
474 s->state & SRV_BACKUP ? "Backup " : "",
475 s->proxy->id, s->id);
476
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200477 server_status_printf(&msg, s,
478 (s->tracked?SSP_O_VIA:0) | SSP_O_STATUS |
479 ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
480 -1);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100481
482 Warning("%s", trash);
483 send_log(s->proxy, LOG_NOTICE, "%s", trash);
484
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100485 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100486 for(srv = s->tracknext; srv; srv = srv->tracknext)
487 set_server_enabled(srv);
488}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200489
490/*
491 * This function is used only for server health-checks. It handles
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200492 * the connection acknowledgement. If the proxy requires L7 health-checks,
493 * it sends the request. In other cases, it calls set_server_check_status()
494 * to set s->check_status, s->check_duration and s->result.
Willy Tarreau83749182007-04-15 20:56:27 +0200495 * The function itself returns 0 if it needs some polling before being called
496 * again, otherwise 1.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200497 */
Willy Tarreau83749182007-04-15 20:56:27 +0200498static int event_srv_chk_w(int fd)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200499{
Willy Tarreau6996e152007-04-30 14:37:43 +0200500 __label__ out_wakeup, out_nowake, out_poll, out_error;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200501 struct task *t = fdtab[fd].owner;
502 struct server *s = t->context;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200503
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +0100504 //fprintf(stderr, "event_srv_chk_w, state=%ld\n", unlikely(fdtab[fd].state));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200505 if (unlikely(fdtab[fd].state == FD_STERROR || (fdtab[fd].ev & FD_POLL_ERR))) {
506 set_server_check_status(s, HCHK_STATUS_L4CON);
Willy Tarreau6996e152007-04-30 14:37:43 +0200507 goto out_error;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200508 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200509
510 /* here, we know that the connection is established */
Willy Tarreau83749182007-04-15 20:56:27 +0200511
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100512 if (!(s->result & SRV_CHK_ERROR)) {
Willy Tarreaubaaee002006-06-26 02:48:02 +0200513 /* we don't want to mark 'UP' a server on which we detected an error earlier */
Willy Tarreauf3c69202006-07-09 16:42:34 +0200514 if ((s->proxy->options & PR_O_HTTP_CHK) ||
Willy Tarreau23677902007-05-08 23:50:35 +0200515 (s->proxy->options & PR_O_SSL3_CHK) ||
516 (s->proxy->options & PR_O_SMTP_CHK)) {
Willy Tarreaubaaee002006-06-26 02:48:02 +0200517 int ret;
Willy Tarreauf3c69202006-07-09 16:42:34 +0200518 /* we want to check if this host replies to HTTP or SSLv3 requests
Willy Tarreaubaaee002006-06-26 02:48:02 +0200519 * so we'll send the request, and won't wake the checker up now.
520 */
Willy Tarreauf3c69202006-07-09 16:42:34 +0200521
522 if (s->proxy->options & PR_O_SSL3_CHK) {
523 /* SSL requires that we put Unix time in the request */
Willy Tarreaub7f694f2008-06-22 17:18:02 +0200524 int gmt_time = htonl(date.tv_sec);
Willy Tarreauf3c69202006-07-09 16:42:34 +0200525 memcpy(s->proxy->check_req + 11, &gmt_time, 4);
526 }
527
Willy Tarreaubaaee002006-06-26 02:48:02 +0200528 ret = send(fd, s->proxy->check_req, s->proxy->check_len, MSG_DONTWAIT | MSG_NOSIGNAL);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200529 if (ret == s->proxy->check_len) {
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +0100530 /* we allow up to <timeout.check> if nonzero for a responce */
Willy Tarreau7cd9d942008-12-21 13:00:41 +0100531 if (s->proxy->timeout.check)
532 t->expire = tick_add_ifset(now_ms, s->proxy->timeout.check);
Willy Tarreauf161a342007-04-08 16:59:42 +0200533 EV_FD_SET(fd, DIR_RD); /* prepare for reading reply */
Willy Tarreau83749182007-04-15 20:56:27 +0200534 goto out_nowake;
535 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200536 else if (ret == 0 || errno == EAGAIN)
537 goto out_poll;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200538 else {
539 switch (errno) {
540 case ECONNREFUSED:
541 case ENETUNREACH:
542 set_server_check_status(s, HCHK_STATUS_L4CON);
543 break;
544
545 default:
546 set_server_check_status(s, HCHK_STATUS_SOCKERR);
547 }
548
Willy Tarreau6996e152007-04-30 14:37:43 +0200549 goto out_error;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200550 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200551 }
552 else {
Willy Tarreau6996e152007-04-30 14:37:43 +0200553 /* We have no data to send to check the connection, and
554 * getsockopt() will not inform us whether the connection
555 * is still pending. So we'll reuse connect() to check the
556 * state of the socket. This has the advantage of givig us
557 * the following info :
558 * - error
559 * - connecting (EALREADY, EINPROGRESS)
560 * - connected (EISCONN, 0)
561 */
562
563 struct sockaddr_in sa;
564
565 sa = (s->check_addr.sin_addr.s_addr) ? s->check_addr : s->addr;
566 sa.sin_port = htons(s->check_port);
567
568 if (connect(fd, (struct sockaddr *)&sa, sizeof(sa)) == 0)
569 errno = 0;
570
571 if (errno == EALREADY || errno == EINPROGRESS)
572 goto out_poll;
573
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200574 if (errno && errno != EISCONN) {
575 set_server_check_status(s, HCHK_STATUS_L4CON);
Willy Tarreau6996e152007-04-30 14:37:43 +0200576 goto out_error;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200577 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200578
Willy Tarreaubaaee002006-06-26 02:48:02 +0200579 /* good TCP connection is enough */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200580 set_server_check_status(s, HCHK_STATUS_L4OK);
Willy Tarreau6996e152007-04-30 14:37:43 +0200581 goto out_wakeup;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200582 }
583 }
Willy Tarreau83749182007-04-15 20:56:27 +0200584 out_wakeup:
Willy Tarreaufdccded2008-08-29 18:19:04 +0200585 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreau83749182007-04-15 20:56:27 +0200586 out_nowake:
587 EV_FD_CLR(fd, DIR_WR); /* nothing more to write */
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100588 fdtab[fd].ev &= ~FD_POLL_OUT;
Willy Tarreau83749182007-04-15 20:56:27 +0200589 return 1;
Willy Tarreau6996e152007-04-30 14:37:43 +0200590 out_poll:
591 /* The connection is still pending. We'll have to poll it
592 * before attempting to go further. */
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100593 fdtab[fd].ev &= ~FD_POLL_OUT;
Willy Tarreau6996e152007-04-30 14:37:43 +0200594 return 0;
595 out_error:
Willy Tarreau6996e152007-04-30 14:37:43 +0200596 fdtab[fd].state = FD_STERROR;
597 goto out_wakeup;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200598}
599
600
601/*
Willy Tarreauf3c69202006-07-09 16:42:34 +0200602 * This function is used only for server health-checks. It handles the server's
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200603 * reply to an HTTP request or SSL HELLO. It calls set_server_check_status() to
604 * update s->check_status, s->check_duration and s->result.
605
606 * The set_server_check_status function is called with HCHK_STATUS_L7OKD if
607 * an HTTP server replies HTTP 2xx or 3xx (valid responses), if an SMTP server
608 * returns 2xx, HCHK_STATUS_L6OK if an SSL server returns at least 5 bytes in
609 * response to an SSL HELLO (the principle is that this is enough to
610 * distinguish between an SSL server and a pure TCP relay). All other cases will
611 * call it with a proper error status like HCHK_STATUS_L7STS, HCHK_STATUS_L6RSP,
612 * etc.
613 *
614 * The function returns 0 if it needs to be called again after some polling,
615 * otherwise non-zero..
Willy Tarreaubaaee002006-06-26 02:48:02 +0200616 */
Willy Tarreau83749182007-04-15 20:56:27 +0200617static int event_srv_chk_r(int fd)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200618{
Willy Tarreau83749182007-04-15 20:56:27 +0200619 __label__ out_wakeup;
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100620 int len;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200621 struct task *t = fdtab[fd].owner;
622 struct server *s = t->context;
623 int skerr;
624 socklen_t lskerr = sizeof(skerr);
625
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100626 len = -1;
Willy Tarreau83749182007-04-15 20:56:27 +0200627
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100628 if (unlikely((s->result & SRV_CHK_ERROR) ||
629 (fdtab[fd].state == FD_STERROR) ||
Willy Tarreau83749182007-04-15 20:56:27 +0200630 (fdtab[fd].ev & FD_POLL_ERR) ||
631 (getsockopt(fd, SOL_SOCKET, SO_ERROR, &skerr, &lskerr) == -1) ||
632 (skerr != 0))) {
633 /* in case of TCP only, this tells us if the connection failed */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200634
635 if (!(s->result & SRV_CHK_ERROR))
636 set_server_check_status(s, HCHK_STATUS_SOCKERR);
637
Willy Tarreau83749182007-04-15 20:56:27 +0200638 goto out_wakeup;
639 }
640
Willy Tarreau83749182007-04-15 20:56:27 +0200641 /* Warning! Linux returns EAGAIN on SO_ERROR if data are still available
642 * but the connection was closed on the remote end. Fortunately, recv still
643 * works correctly and we don't need to do the getsockopt() on linux.
644 */
Krzysztof Oledzki6b3f8b42007-10-11 18:41:08 +0200645 len = recv(fd, trash, sizeof(trash), MSG_NOSIGNAL);
Willy Tarreau83749182007-04-15 20:56:27 +0200646 if (unlikely(len < 0 && errno == EAGAIN)) {
647 /* we want some polling to happen first */
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100648 fdtab[fd].ev &= ~FD_POLL_IN;
Willy Tarreau83749182007-04-15 20:56:27 +0200649 return 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200650 }
651
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100652 /* Note: the response will only be accepted if read at once */
653 if (s->proxy->options & PR_O_HTTP_CHK) {
654 /* Check if the server speaks HTTP 1.X */
655 if ((len < strlen("HTTP/1.0 000\r")) ||
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200656 (memcmp(trash, "HTTP/1.", 7) != 0 ||
657 (trash[12] != ' ' && trash[12] != '\r')) ||
658 !isdigit(trash[9]) || !isdigit(trash[10]) || !isdigit(trash[11])) {
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200659 set_server_check_status(s, HCHK_STATUS_L7RSP);
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100660 goto out_wakeup;
661 }
662
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200663 s->check_code = str2uic(&trash[9]);
664
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100665 /* check the reply : HTTP/1.X 2xx and 3xx are OK */
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200666 if (trash[9] == '2' || trash[9] == '3')
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200667 set_server_check_status(s, HCHK_STATUS_L7OKD);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200668 else if ((s->proxy->options & PR_O_DISABLE404) &&
Willy Tarreau48494c02007-11-30 10:41:39 +0100669 (s->state & SRV_RUNNING) &&
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200670 (s->check_code == 404))
671 /* 404 may be accepted as "stopping" only if the server was up */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200672 set_server_check_status(s, HCHK_STATUS_L7OKCD);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200673 else
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200674 set_server_check_status(s, HCHK_STATUS_L7STS);
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100675 }
676 else if (s->proxy->options & PR_O_SSL3_CHK) {
677 /* Check for SSLv3 alert or handshake */
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200678 if ((len >= 5) && (trash[0] == 0x15 || trash[0] == 0x16))
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200679 set_server_check_status(s, HCHK_STATUS_L6OK);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200680 else
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200681 set_server_check_status(s, HCHK_STATUS_L6RSP);
Willy Tarreau6996e152007-04-30 14:37:43 +0200682 }
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100683 else if (s->proxy->options & PR_O_SMTP_CHK) {
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200684 /* Check if the server speaks SMTP */
685 if ((len < strlen("000\r")) ||
686 (trash[3] != ' ' && trash[3] != '\r') ||
687 !isdigit(trash[0]) || !isdigit(trash[1]) || !isdigit(trash[2])) {
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200688 set_server_check_status(s, HCHK_STATUS_L7RSP);
689 goto out_wakeup;
690 }
691
692 s->check_code = str2uic(&trash[0]);
693
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100694 /* Check for SMTP code 2xx (should be 250) */
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200695 if (trash[0] == '2')
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200696 set_server_check_status(s, HCHK_STATUS_L7OKD);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200697 else
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200698 set_server_check_status(s, HCHK_STATUS_L7STS);
Willy Tarreau6996e152007-04-30 14:37:43 +0200699 }
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100700 else {
701 /* other checks are valid if the connection succeeded anyway */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200702 set_server_check_status(s, HCHK_STATUS_L4OK);
Willy Tarreau23677902007-05-08 23:50:35 +0200703 }
Willy Tarreau83749182007-04-15 20:56:27 +0200704
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100705 out_wakeup:
706 if (s->result & SRV_CHK_ERROR)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200707 fdtab[fd].state = FD_STERROR;
708
Willy Tarreauf161a342007-04-08 16:59:42 +0200709 EV_FD_CLR(fd, DIR_RD);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200710 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100711 fdtab[fd].ev &= ~FD_POLL_IN;
Willy Tarreau83749182007-04-15 20:56:27 +0200712 return 1;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200713}
714
715/*
716 * manages a server health-check. Returns
717 * the time the task accepts to wait, or TIME_ETERNITY for infinity.
718 */
Willy Tarreau26c25062009-03-08 09:38:41 +0100719struct task *process_chk(struct task *t)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200720{
Willy Tarreaue3838802009-03-21 18:58:32 +0100721 int attempts = 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200722 struct server *s = t->context;
723 struct sockaddr_in sa;
724 int fd;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +0200725 int rv;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200726
727 //fprintf(stderr, "process_chk: task=%p\n", t);
728
729 new_chk:
Willy Tarreaue3838802009-03-21 18:58:32 +0100730 if (attempts++ > 0) {
731 /* we always fail to create a server, let's stop insisting... */
732 while (tick_is_expired(t->expire, now_ms))
733 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
734 return t;
735 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200736 fd = s->curfd;
737 if (fd < 0) { /* no check currently running */
738 //fprintf(stderr, "process_chk: 2\n");
Willy Tarreau26c25062009-03-08 09:38:41 +0100739 if (!tick_is_expired(t->expire, now_ms)) /* woke up too early */
740 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200741
742 /* we don't send any health-checks when the proxy is stopped or when
743 * the server should not be checked.
744 */
745 if (!(s->state & SRV_CHECKED) || s->proxy->state == PR_STSTOPPED) {
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200746 while (tick_is_expired(t->expire, now_ms))
747 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
Willy Tarreau26c25062009-03-08 09:38:41 +0100748 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200749 }
750
751 /* we'll initiate a new check */
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200752 set_server_check_status(s, HCHK_STATUS_START);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200753 if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) != -1) {
754 if ((fd < global.maxsock) &&
755 (fcntl(fd, F_SETFL, O_NONBLOCK) != -1) &&
756 (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char *) &one, sizeof(one)) != -1)) {
757 //fprintf(stderr, "process_chk: 3\n");
758
Willy Tarreau9edd1612007-10-18 18:07:48 +0200759 if (s->proxy->options & PR_O_TCP_NOLING) {
760 /* We don't want to useless data */
761 setsockopt(fd, SOL_SOCKET, SO_LINGER, (struct linger *) &nolinger, sizeof(struct linger));
762 }
Willy Tarreau2ea3abb2007-03-25 16:45:16 +0200763
Willy Tarreau0f03c6f2007-03-25 20:46:19 +0200764 if (s->check_addr.sin_addr.s_addr)
765 /* we'll connect to the check addr specified on the server */
Willy Tarreau2ea3abb2007-03-25 16:45:16 +0200766 sa = s->check_addr;
Willy Tarreau2ea3abb2007-03-25 16:45:16 +0200767 else
Willy Tarreau0f03c6f2007-03-25 20:46:19 +0200768 /* we'll connect to the addr on the server */
Willy Tarreau2ea3abb2007-03-25 16:45:16 +0200769 sa = s->addr;
Willy Tarreau0f03c6f2007-03-25 20:46:19 +0200770
Willy Tarreaubaaee002006-06-26 02:48:02 +0200771 /* we'll connect to the check port on the server */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200772 sa.sin_port = htons(s->check_port);
773
774 /* allow specific binding :
775 * - server-specific at first
776 * - proxy-specific next
777 */
778 if (s->state & SRV_BIND_SRC) {
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100779 struct sockaddr_in *remote = NULL;
780 int ret, flags = 0;
Willy Tarreau163c5322006-11-14 16:18:41 +0100781
Willy Tarreaucf1d5722008-02-14 20:28:18 +0100782#if defined(CONFIG_HAP_CTTPROXY) || defined(CONFIG_HAP_LINUX_TPROXY)
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100783 if ((s->state & SRV_TPROXY_MASK) == SRV_TPROXY_ADDR) {
784 remote = (struct sockaddr_in *)&s->tproxy_addr;
785 flags = 3;
786 }
Willy Tarreaucf1d5722008-02-14 20:28:18 +0100787#endif
Willy Tarreauc76721d2009-02-04 20:20:58 +0100788#ifdef SO_BINDTODEVICE
789 /* Note: this might fail if not CAP_NET_RAW */
790 if (s->iface_name)
791 setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE,
Willy Tarreau604e8302009-03-06 00:48:23 +0100792 s->iface_name, s->iface_len + 1);
Willy Tarreauc76721d2009-02-04 20:20:58 +0100793#endif
Willy Tarreauc6f4ce82009-06-10 11:09:37 +0200794 if (s->sport_range) {
795 int bind_attempts = 10; /* should be more than enough to find a spare port */
796 struct sockaddr_in src;
797
798 ret = 1;
799 src = s->source_addr;
800
801 do {
802 /* note: in case of retry, we may have to release a previously
803 * allocated port, hence this loop's construct.
804 */
805 port_range_release_port(fdtab[fd].port_range, fdtab[fd].local_port);
806 fdtab[fd].port_range = NULL;
807
808 if (!bind_attempts)
809 break;
810 bind_attempts--;
811
812 fdtab[fd].local_port = port_range_alloc_port(s->sport_range);
813 if (!fdtab[fd].local_port)
814 break;
815
816 fdtab[fd].port_range = s->sport_range;
817 src.sin_port = htons(fdtab[fd].local_port);
818
819 ret = tcpv4_bind_socket(fd, flags, &src, remote);
820 } while (ret != 0); /* binding NOK */
821 }
822 else {
823 ret = tcpv4_bind_socket(fd, flags, &s->source_addr, remote);
824 }
825
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100826 if (ret) {
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200827 set_server_check_status(s, HCHK_STATUS_SOCKERR);
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100828 switch (ret) {
829 case 1:
830 Alert("Cannot bind to source address before connect() for server %s/%s. Aborting.\n",
831 s->proxy->id, s->id);
832 break;
833 case 2:
Willy Tarreau163c5322006-11-14 16:18:41 +0100834 Alert("Cannot bind to tproxy source address before connect() for server %s/%s. Aborting.\n",
835 s->proxy->id, s->id);
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100836 break;
Willy Tarreau163c5322006-11-14 16:18:41 +0100837 }
838 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200839 }
840 else if (s->proxy->options & PR_O_BIND_SRC) {
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100841 struct sockaddr_in *remote = NULL;
842 int ret, flags = 0;
843
Willy Tarreaucf1d5722008-02-14 20:28:18 +0100844#if defined(CONFIG_HAP_CTTPROXY) || defined(CONFIG_HAP_LINUX_TPROXY)
Willy Tarreau163c5322006-11-14 16:18:41 +0100845 if ((s->proxy->options & PR_O_TPXY_MASK) == PR_O_TPXY_ADDR) {
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100846 remote = (struct sockaddr_in *)&s->proxy->tproxy_addr;
847 flags = 3;
848 }
Willy Tarreaucf1d5722008-02-14 20:28:18 +0100849#endif
Willy Tarreaud53f96b2009-02-04 18:46:54 +0100850#ifdef SO_BINDTODEVICE
851 /* Note: this might fail if not CAP_NET_RAW */
852 if (s->proxy->iface_name)
853 setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE,
Willy Tarreau604e8302009-03-06 00:48:23 +0100854 s->proxy->iface_name, s->proxy->iface_len + 1);
Willy Tarreaud53f96b2009-02-04 18:46:54 +0100855#endif
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100856 ret = tcpv4_bind_socket(fd, flags, &s->proxy->source_addr, remote);
857 if (ret) {
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200858 set_server_check_status(s, HCHK_STATUS_SOCKERR);
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100859 switch (ret) {
860 case 1:
861 Alert("Cannot bind to source address before connect() for %s '%s'. Aborting.\n",
862 proxy_type_str(s->proxy), s->proxy->id);
863 break;
864 case 2:
Willy Tarreau2b5652f2006-12-31 17:46:05 +0100865 Alert("Cannot bind to tproxy source address before connect() for %s '%s'. Aborting.\n",
866 proxy_type_str(s->proxy), s->proxy->id);
Willy Tarreaue8c66af2008-01-13 18:40:14 +0100867 break;
Willy Tarreau163c5322006-11-14 16:18:41 +0100868 }
869 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200870 }
871
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100872 if (s->result == SRV_CHK_UNKNOWN) {
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +0400873#if defined(TCP_QUICKACK)
Willy Tarreau1274bc42009-07-15 07:16:31 +0200874 /* disabling tcp quick ack now allows
875 * the request to leave the machine with
876 * the first ACK.
877 */
878 if (s->proxy->options2 & PR_O2_SMARTCON)
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +0400879 setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, (char *) &zero, sizeof(zero));
Willy Tarreau1274bc42009-07-15 07:16:31 +0200880#endif
Willy Tarreaubaaee002006-06-26 02:48:02 +0200881 if ((connect(fd, (struct sockaddr *)&sa, sizeof(sa)) != -1) || (errno == EINPROGRESS)) {
882 /* OK, connection in progress or established */
883
884 //fprintf(stderr, "process_chk: 4\n");
885
886 s->curfd = fd; /* that's how we know a test is in progress ;-) */
Willy Tarreau7a966482007-04-15 10:58:02 +0200887 fd_insert(fd);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200888 fdtab[fd].owner = t;
Willy Tarreau54469402006-07-29 16:59:06 +0200889 fdtab[fd].cb[DIR_RD].f = &event_srv_chk_r;
890 fdtab[fd].cb[DIR_RD].b = NULL;
891 fdtab[fd].cb[DIR_WR].f = &event_srv_chk_w;
892 fdtab[fd].cb[DIR_WR].b = NULL;
Willy Tarreaue94ebd02007-10-09 17:14:37 +0200893 fdtab[fd].peeraddr = (struct sockaddr *)&sa;
894 fdtab[fd].peerlen = sizeof(sa);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200895 fdtab[fd].state = FD_STCONN; /* connection in progress */
Willy Tarreaufb14edc2009-06-14 15:24:37 +0200896 fdtab[fd].flags = FD_FL_TCP | FD_FL_TCP_NODELAY;
Willy Tarreauf161a342007-04-08 16:59:42 +0200897 EV_FD_SET(fd, DIR_WR); /* for connect status */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200898#ifdef DEBUG_FULL
Willy Tarreauf161a342007-04-08 16:59:42 +0200899 assert (!EV_FD_ISSET(fd, DIR_RD));
Willy Tarreaubaaee002006-06-26 02:48:02 +0200900#endif
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +0100901 //fprintf(stderr, "process_chk: 4+, %lu\n", __tv_to_ms(&s->proxy->timeout.connect));
902 /* we allow up to min(inter, timeout.connect) for a connection
903 * to establish but only when timeout.check is set
904 * as it may be to short for a full check otherwise
905 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200906 t->expire = tick_add(now_ms, MS_TO_TICKS(s->inter));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +0100907
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200908 if (s->proxy->timeout.check && s->proxy->timeout.connect) {
909 int t_con = tick_add(now_ms, s->proxy->timeout.connect);
910 t->expire = tick_first(t->expire, t_con);
Willy Tarreau60548192008-02-17 11:34:10 +0100911 }
Willy Tarreau26c25062009-03-08 09:38:41 +0100912 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200913 }
914 else if (errno != EALREADY && errno != EISCONN && errno != EAGAIN) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200915 /* a real error */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200916
917 switch (errno) {
918 /* FIXME: is it possible to get ECONNREFUSED/ENETUNREACH with O_NONBLOCK? */
919 case ECONNREFUSED:
920 case ENETUNREACH:
921 set_server_check_status(s, HCHK_STATUS_L4CON);
922 break;
923
924 default:
925 set_server_check_status(s, HCHK_STATUS_SOCKERR);
926 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200927 }
928 }
929 }
Willy Tarreauc6f4ce82009-06-10 11:09:37 +0200930 port_range_release_port(fdtab[fd].port_range, fdtab[fd].local_port);
931 fdtab[fd].port_range = NULL;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200932 close(fd); /* socket creation error */
933 }
934
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100935 if (s->result == SRV_CHK_UNKNOWN) { /* nothing done */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200936 //fprintf(stderr, "process_chk: 6\n");
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200937 while (tick_is_expired(t->expire, now_ms))
938 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
Willy Tarreaubaaee002006-06-26 02:48:02 +0200939 goto new_chk; /* may be we should initialize a new check */
940 }
941
942 /* here, we have seen a failure */
943 if (s->health > s->rise) {
944 s->health--; /* still good */
945 s->failed_checks++;
946 }
947 else
948 set_server_down(s);
949
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +0100950 //fprintf(stderr, "process_chk: 7, %lu\n", __tv_to_ms(&s->proxy->timeout.connect));
951 /* we allow up to min(inter, timeout.connect) for a connection
952 * to establish but only when timeout.check is set
953 * as it may be to short for a full check otherwise
954 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200955 while (tick_is_expired(t->expire, now_ms)) {
956 int t_con;
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +0100957
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200958 t_con = tick_add(t->expire, s->proxy->timeout.connect);
959 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +0100960
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200961 if (s->proxy->timeout.check)
962 t->expire = tick_first(t->expire, t_con);
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +0100963 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200964 goto new_chk;
965 }
966 else {
967 //fprintf(stderr, "process_chk: 8\n");
968 /* there was a test running */
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100969 if ((s->result & (SRV_CHK_ERROR|SRV_CHK_RUNNING)) == SRV_CHK_RUNNING) { /* good server detected */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200970 //fprintf(stderr, "process_chk: 9\n");
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200971
Willy Tarreau9909fc12007-11-30 17:42:05 +0100972 if (s->state & SRV_WARMINGUP) {
973 if (now.tv_sec < s->last_change || now.tv_sec >= s->last_change + s->slowstart) {
974 s->state &= ~SRV_WARMINGUP;
975 if (s->proxy->lbprm.algo & BE_LB_PROP_DYN)
976 s->eweight = s->uweight * BE_WEIGHT_SCALE;
977 if (s->proxy->lbprm.update_server_eweight)
978 s->proxy->lbprm.update_server_eweight(s);
979 }
980 else if (s->proxy->lbprm.algo & BE_LB_PROP_DYN) {
981 /* for dynamic algorithms, let's update the weight */
Willy Tarreau5542af62007-12-03 02:04:00 +0100982 s->eweight = (BE_WEIGHT_SCALE * (now.tv_sec - s->last_change) +
983 s->slowstart - 1) / s->slowstart;
Willy Tarreau9909fc12007-11-30 17:42:05 +0100984 s->eweight *= s->uweight;
985 if (s->proxy->lbprm.update_server_eweight)
986 s->proxy->lbprm.update_server_eweight(s);
987 }
988 /* probably that we can refill this server with a bit more connections */
989 check_for_pending(s);
990 }
991
Willy Tarreau48494c02007-11-30 10:41:39 +0100992 /* we may have to add/remove this server from the LB group */
993 if ((s->state & SRV_RUNNING) && (s->proxy->options & PR_O_DISABLE404)) {
994 if ((s->state & SRV_GOINGDOWN) &&
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100995 ((s->result & (SRV_CHK_RUNNING|SRV_CHK_DISABLE)) == SRV_CHK_RUNNING))
996 set_server_enabled(s);
Willy Tarreau48494c02007-11-30 10:41:39 +0100997 else if (!(s->state & SRV_GOINGDOWN) &&
998 ((s->result & (SRV_CHK_RUNNING | SRV_CHK_DISABLE)) ==
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100999 (SRV_CHK_RUNNING | SRV_CHK_DISABLE)))
1000 set_server_disabled(s);
Willy Tarreau48494c02007-11-30 10:41:39 +01001001 }
1002
Krzysztof Oledzki85130942007-10-22 16:21:10 +02001003 if (s->health < s->rise + s->fall - 1) {
1004 s->health++; /* was bad, stays for a while */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001005
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +01001006 set_server_up(s);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001007 }
1008 s->curfd = -1; /* no check running anymore */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001009 fd_delete(fd);
Willy Tarreau44ec0f02007-10-14 23:47:04 +02001010
1011 rv = 0;
1012 if (global.spread_checks > 0) {
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001013 rv = srv_getinter(s) * global.spread_checks / 100;
Willy Tarreau44ec0f02007-10-14 23:47:04 +02001014 rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0)));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001015 //fprintf(stderr, "process_chk(%p): (%d+/-%d%%) random=%d\n", s, srv_getinter(s), global.spread_checks, rv);
Willy Tarreau44ec0f02007-10-14 23:47:04 +02001016 }
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001017 t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(s) + rv));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001018 goto new_chk;
1019 }
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001020 else if ((s->result & SRV_CHK_ERROR) || tick_is_expired(t->expire, now_ms)) {
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001021 if (!(s->result & SRV_CHK_ERROR)) {
1022 if (!EV_FD_ISSET(fd, DIR_RD)) {
1023 set_server_check_status(s, HCHK_STATUS_L4TOUT);
1024 } else {
1025 if (s->proxy->options & PR_O_SSL3_CHK)
1026 set_server_check_status(s, HCHK_STATUS_L6TOUT);
1027 else /* HTTP, SMTP */
1028 set_server_check_status(s, HCHK_STATUS_L7TOUT);
1029 }
1030 }
1031
Willy Tarreaubaaee002006-06-26 02:48:02 +02001032 //fprintf(stderr, "process_chk: 10\n");
1033 /* failure or timeout detected */
1034 if (s->health > s->rise) {
1035 s->health--; /* still good */
1036 s->failed_checks++;
1037 }
1038 else
1039 set_server_down(s);
1040 s->curfd = -1;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001041 fd_delete(fd);
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001042
1043 rv = 0;
1044 if (global.spread_checks > 0) {
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001045 rv = srv_getinter(s) * global.spread_checks / 100;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001046 rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0)));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001047 //fprintf(stderr, "process_chk(%p): (%d+/-%d%%) random=%d\n", s, srv_getinter(s), global.spread_checks, rv);
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001048 }
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001049 t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(s) + rv));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001050 goto new_chk;
1051 }
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001052 /* if result is unknown and there's no timeout, we have to wait again */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001053 }
1054 //fprintf(stderr, "process_chk: 11\n");
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001055 s->result = SRV_CHK_UNKNOWN;
Willy Tarreau26c25062009-03-08 09:38:41 +01001056 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001057}
1058
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001059/*
1060 * Start health-check.
1061 * Returns 0 if OK, -1 if error, and prints the error in this case.
1062 */
1063int start_checks() {
1064
1065 struct proxy *px;
1066 struct server *s;
1067 struct task *t;
1068 int nbchk=0, mininter=0, srvpos=0;
1069
Willy Tarreau2c43a1e2007-10-14 23:05:39 +02001070 /* 1- count the checkers to run simultaneously.
1071 * We also determine the minimum interval among all of those which
1072 * have an interval larger than SRV_CHK_INTER_THRES. This interval
1073 * will be used to spread their start-up date. Those which have
1074 * a shorter interval will start independantly and will not dictate
1075 * too short an interval for all others.
1076 */
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001077 for (px = proxy; px; px = px->next) {
1078 for (s = px->srv; s; s = s->next) {
1079 if (!(s->state & SRV_CHECKED))
1080 continue;
1081
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001082 if ((srv_getinter(s) >= SRV_CHK_INTER_THRES) &&
1083 (!mininter || mininter > srv_getinter(s)))
1084 mininter = srv_getinter(s);
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001085
1086 nbchk++;
1087 }
1088 }
1089
1090 if (!nbchk)
1091 return 0;
1092
1093 srand((unsigned)time(NULL));
1094
1095 /*
1096 * 2- start them as far as possible from each others. For this, we will
1097 * start them after their interval set to the min interval divided by
1098 * the number of servers, weighted by the server's position in the list.
1099 */
1100 for (px = proxy; px; px = px->next) {
1101 for (s = px->srv; s; s = s->next) {
1102 if (!(s->state & SRV_CHECKED))
1103 continue;
1104
Willy Tarreaua4613182009-03-21 18:13:21 +01001105 if ((t = task_new()) == NULL) {
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001106 Alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
1107 return -1;
1108 }
1109
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +02001110 s->check = t;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001111 t->process = process_chk;
1112 t->context = s;
1113
1114 /* check this every ms */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001115 t->expire = tick_add(now_ms,
1116 MS_TO_TICKS(((mininter && mininter >= srv_getinter(s)) ?
1117 mininter : srv_getinter(s)) * srvpos / nbchk));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001118 s->check_start = now;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001119 task_queue(t);
1120
1121 srvpos++;
1122 }
1123 }
1124 return 0;
1125}
Willy Tarreaubaaee002006-06-26 02:48:02 +02001126
1127/*
1128 * Local variables:
1129 * c-indent-level: 8
1130 * c-basic-offset: 8
1131 * End:
1132 */