blob: 456d7a3931c81e7eeaf3290874068e17a9c48be2 [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
2 * Health-checks functions.
3 *
4 * Copyright 2000-2006 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
13#include <errno.h>
14#include <fcntl.h>
15#include <stdio.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020016#include <string.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020017#include <unistd.h>
18#include <sys/socket.h>
19#include <netinet/in.h>
20#include <arpa/inet.h>
21
Willy Tarreau2dd0d472006-06-29 17:53:05 +020022#include <common/compat.h>
23#include <common/config.h>
24#include <common/mini-clist.h>
25#include <common/time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020026
27#include <types/global.h>
28#include <types/polling.h>
29#include <types/proxy.h>
30#include <types/session.h>
31
32#include <proto/backend.h>
33#include <proto/fd.h>
34#include <proto/log.h>
35#include <proto/queue.h>
Willy Tarreau3d300592007-03-18 18:34:41 +010036#include <proto/proto_http.h>
Willy Tarreau2b5652f2006-12-31 17:46:05 +010037#include <proto/proxy.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020038#include <proto/server.h>
39#include <proto/task.h>
40
Willy Tarreau163c5322006-11-14 16:18:41 +010041#ifdef CONFIG_HAP_CTTPROXY
42#include <import/ip_tproxy.h>
43#endif
44
Willy Tarreaubaaee002006-06-26 02:48:02 +020045
46/* Sets server <s> down, notifies by all available means, recounts the
47 * remaining servers on the proxy and transfers queued sessions whenever
48 * possible to other servers.
49 */
50void set_server_down(struct server *s)
51{
52 struct pendconn *pc, *pc_bck, *pc_end;
53 struct session *sess;
54 int xferred;
55
56 s->state &= ~SRV_RUNNING;
57
58 if (s->health == s->rise) {
59 recount_servers(s->proxy);
60 recalc_server_map(s->proxy);
61
62 /* we might have sessions queued on this server and waiting for
63 * a connection. Those which are redispatchable will be queued
64 * to another server or to the proxy itself.
65 */
66 xferred = 0;
67 FOREACH_ITEM_SAFE(pc, pc_bck, &s->pendconns, pc_end, struct pendconn *, list) {
68 sess = pc->sess;
Willy Tarreaue2e27a52007-04-01 00:01:37 +020069 if ((sess->be->options & PR_O_REDISP)) {
Willy Tarreaubaaee002006-06-26 02:48:02 +020070 /* The REDISP option was specified. We will ignore
71 * cookie and force to balance or use the dispatcher.
72 */
73 sess->flags &= ~(SN_DIRECT | SN_ASSIGNED | SN_ADDR_SET);
74 sess->srv = NULL; /* it's left to the dispatcher to choose a server */
Willy Tarreau3d300592007-03-18 18:34:41 +010075 http_flush_cookie_flags(&sess->txn);
Willy Tarreaubaaee002006-06-26 02:48:02 +020076 pendconn_free(pc);
77 task_wakeup(&rq, sess->task);
78 xferred++;
79 }
80 }
81
82 sprintf(trash, "%sServer %s/%s is DOWN. %d active and %d backup servers left.%s"
83 " %d sessions active, %d requeued, %d remaining in queue.\n",
84 s->state & SRV_BACKUP ? "Backup " : "",
85 s->proxy->id, s->id, s->proxy->srv_act, s->proxy->srv_bck,
86 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
87 s->cur_sess, xferred, s->nbpend);
88
89 Warning("%s", trash);
90 send_log(s->proxy, LOG_ALERT, "%s", trash);
91
92 if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0) {
Willy Tarreau2b5652f2006-12-31 17:46:05 +010093 Alert("%s '%s' has no server available !\n", proxy_type_str(s->proxy), s->proxy->id);
94 send_log(s->proxy, LOG_EMERG, "%s %s has no server available !\n", proxy_type_str(s->proxy), s->proxy->id);
Willy Tarreaubaaee002006-06-26 02:48:02 +020095 }
96 s->down_trans++;
97 }
98 s->health = 0; /* failure */
99}
100
101
102/*
103 * This function is used only for server health-checks. It handles
104 * the connection acknowledgement. If the proxy requires HTTP health-checks,
105 * it sends the request. In other cases, it returns 1 if the socket is OK,
106 * or -1 if an error occured.
107 */
108int event_srv_chk_w(int fd)
109{
110 struct task *t = fdtab[fd].owner;
111 struct server *s = t->context;
112 int skerr;
113 socklen_t lskerr = sizeof(skerr);
114
115 skerr = 1;
116 if ((getsockopt(fd, SOL_SOCKET, SO_ERROR, &skerr, &lskerr) == -1)
117 || (skerr != 0)) {
118 /* in case of TCP only, this tells us if the connection failed */
119 s->result = -1;
120 fdtab[fd].state = FD_STERROR;
Willy Tarreau2a429502006-10-15 14:52:29 +0200121 MY_FD_CLR(fd, StaticWriteEvent);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200122 }
123 else if (s->result != -1) {
124 /* we don't want to mark 'UP' a server on which we detected an error earlier */
Willy Tarreauf3c69202006-07-09 16:42:34 +0200125 if ((s->proxy->options & PR_O_HTTP_CHK) ||
126 (s->proxy->options & PR_O_SSL3_CHK)) {
Willy Tarreaubaaee002006-06-26 02:48:02 +0200127 int ret;
Willy Tarreauf3c69202006-07-09 16:42:34 +0200128 /* we want to check if this host replies to HTTP or SSLv3 requests
Willy Tarreaubaaee002006-06-26 02:48:02 +0200129 * so we'll send the request, and won't wake the checker up now.
130 */
Willy Tarreauf3c69202006-07-09 16:42:34 +0200131
132 if (s->proxy->options & PR_O_SSL3_CHK) {
133 /* SSL requires that we put Unix time in the request */
134 int gmt_time = htonl(now.tv_sec);
135 memcpy(s->proxy->check_req + 11, &gmt_time, 4);
136 }
137
Willy Tarreaubaaee002006-06-26 02:48:02 +0200138#ifndef MSG_NOSIGNAL
139 ret = send(fd, s->proxy->check_req, s->proxy->check_len, MSG_DONTWAIT);
140#else
141 ret = send(fd, s->proxy->check_req, s->proxy->check_len, MSG_DONTWAIT | MSG_NOSIGNAL);
142#endif
143 if (ret == s->proxy->check_len) {
Willy Tarreau2a429502006-10-15 14:52:29 +0200144 MY_FD_SET(fd, StaticReadEvent); /* prepare for reading reply */
145 MY_FD_CLR(fd, StaticWriteEvent); /* nothing more to write */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200146 return 0;
147 }
148 else {
149 s->result = -1;
Willy Tarreau2a429502006-10-15 14:52:29 +0200150 MY_FD_CLR(fd, StaticWriteEvent);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200151 }
152 }
153 else {
154 /* good TCP connection is enough */
155 s->result = 1;
156 }
157 }
158
159 task_wakeup(&rq, t);
160 return 0;
161}
162
163
164/*
Willy Tarreauf3c69202006-07-09 16:42:34 +0200165 * This function is used only for server health-checks. It handles the server's
166 * reply to an HTTP request or SSL HELLO. It returns 1 in s->result if the
167 * server replies HTTP 2xx or 3xx (valid responses), or if it returns at least
168 * 5 bytes in response to SSL HELLO. The principle is that this is enough to
169 * distinguish between an SSL server and a pure TCP relay. All other cases will
170 * return -1. The function returns 0.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200171 */
172int event_srv_chk_r(int fd)
173{
174 char reply[64];
175 int len, result;
176 struct task *t = fdtab[fd].owner;
177 struct server *s = t->context;
178 int skerr;
179 socklen_t lskerr = sizeof(skerr);
180
181 result = len = -1;
Willy Tarreauc6423482006-10-15 14:59:03 +0200182 if (!getsockopt(fd, SOL_SOCKET, SO_ERROR, &skerr, &lskerr) && !skerr) {
Willy Tarreaubaaee002006-06-26 02:48:02 +0200183#ifndef MSG_NOSIGNAL
184 len = recv(fd, reply, sizeof(reply), 0);
185#else
186 /* Warning! Linux returns EAGAIN on SO_ERROR if data are still available
187 * but the connection was closed on the remote end. Fortunately, recv still
188 * works correctly and we don't need to do the getsockopt() on linux.
189 */
190 len = recv(fd, reply, sizeof(reply), MSG_NOSIGNAL);
191#endif
Willy Tarreauf3c69202006-07-09 16:42:34 +0200192 if (((s->proxy->options & PR_O_HTTP_CHK) &&
193 (len >= sizeof("HTTP/1.0 000")) &&
Willy Tarreaubaaee002006-06-26 02:48:02 +0200194 !memcmp(reply, "HTTP/1.", 7) &&
195 (reply[9] == '2' || reply[9] == '3')) /* 2xx or 3xx */
Willy Tarreauf3c69202006-07-09 16:42:34 +0200196 || ((s->proxy->options & PR_O_SSL3_CHK) && (len >= 5) &&
197 (reply[0] == 0x15 || reply[0] == 0x16))) /* alert or handshake */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200198 result = 1;
199 }
200
201 if (result == -1)
202 fdtab[fd].state = FD_STERROR;
203
204 if (s->result != -1)
205 s->result = result;
206
Willy Tarreau2a429502006-10-15 14:52:29 +0200207 MY_FD_CLR(fd, StaticReadEvent);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200208 task_wakeup(&rq, t);
209 return 0;
210}
211
212/*
213 * manages a server health-check. Returns
214 * the time the task accepts to wait, or TIME_ETERNITY for infinity.
215 */
216int process_chk(struct task *t)
217{
218 struct server *s = t->context;
219 struct sockaddr_in sa;
220 int fd;
221
222 //fprintf(stderr, "process_chk: task=%p\n", t);
223
224 new_chk:
225 fd = s->curfd;
226 if (fd < 0) { /* no check currently running */
227 //fprintf(stderr, "process_chk: 2\n");
228 if (tv_cmp2_ms(&t->expire, &now) > 0) { /* not good time yet */
229 task_queue(t); /* restore t to its place in the task list */
230 return tv_remain2(&now, &t->expire);
231 }
232
233 /* we don't send any health-checks when the proxy is stopped or when
234 * the server should not be checked.
235 */
236 if (!(s->state & SRV_CHECKED) || s->proxy->state == PR_STSTOPPED) {
237 while (tv_cmp2_ms(&t->expire, &now) <= 0)
238 tv_delayfrom(&t->expire, &t->expire, s->inter);
239 task_queue(t); /* restore t to its place in the task list */
240 return tv_remain2(&now, &t->expire);
241 }
242
243 /* we'll initiate a new check */
244 s->result = 0; /* no result yet */
245 if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) != -1) {
246 if ((fd < global.maxsock) &&
247 (fcntl(fd, F_SETFL, O_NONBLOCK) != -1) &&
248 (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char *) &one, sizeof(one)) != -1)) {
249 //fprintf(stderr, "process_chk: 3\n");
250
Willy Tarreau2ea3abb2007-03-25 16:45:16 +0200251
Willy Tarreau0f03c6f2007-03-25 20:46:19 +0200252 if (s->check_addr.sin_addr.s_addr)
253 /* we'll connect to the check addr specified on the server */
Willy Tarreau2ea3abb2007-03-25 16:45:16 +0200254 sa = s->check_addr;
Willy Tarreau2ea3abb2007-03-25 16:45:16 +0200255 else
Willy Tarreau0f03c6f2007-03-25 20:46:19 +0200256 /* we'll connect to the addr on the server */
Willy Tarreau2ea3abb2007-03-25 16:45:16 +0200257 sa = s->addr;
Willy Tarreau0f03c6f2007-03-25 20:46:19 +0200258
Willy Tarreaubaaee002006-06-26 02:48:02 +0200259 /* we'll connect to the check port on the server */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200260 sa.sin_port = htons(s->check_port);
261
262 /* allow specific binding :
263 * - server-specific at first
264 * - proxy-specific next
265 */
266 if (s->state & SRV_BIND_SRC) {
267 setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *) &one, sizeof(one));
268 if (bind(fd, (struct sockaddr *)&s->source_addr, sizeof(s->source_addr)) == -1) {
269 Alert("Cannot bind to source address before connect() for server %s/%s. Aborting.\n",
270 s->proxy->id, s->id);
271 s->result = -1;
272 }
Willy Tarreau163c5322006-11-14 16:18:41 +0100273#ifdef CONFIG_HAP_CTTPROXY
274 if ((s->state & SRV_TPROXY_MASK) == SRV_TPROXY_ADDR) {
275 struct in_tproxy itp1, itp2;
276 memset(&itp1, 0, sizeof(itp1));
277
278 itp1.op = TPROXY_ASSIGN;
279 itp1.v.addr.faddr = s->tproxy_addr.sin_addr;
280 itp1.v.addr.fport = s->tproxy_addr.sin_port;
281
282 /* set connect flag on socket */
283 itp2.op = TPROXY_FLAGS;
284 itp2.v.flags = ITP_CONNECT | ITP_ONCE;
285
286 if (setsockopt(fd, SOL_IP, IP_TPROXY, &itp1, sizeof(itp1)) == -1 ||
287 setsockopt(fd, SOL_IP, IP_TPROXY, &itp2, sizeof(itp2)) == -1) {
288 Alert("Cannot bind to tproxy source address before connect() for server %s/%s. Aborting.\n",
289 s->proxy->id, s->id);
290 s->result = -1;
291 }
292 }
293#endif
Willy Tarreaubaaee002006-06-26 02:48:02 +0200294 }
295 else if (s->proxy->options & PR_O_BIND_SRC) {
296 setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (char *) &one, sizeof(one));
297 if (bind(fd, (struct sockaddr *)&s->proxy->source_addr, sizeof(s->proxy->source_addr)) == -1) {
Willy Tarreau2b5652f2006-12-31 17:46:05 +0100298 Alert("Cannot bind to source address before connect() for %s '%s'. Aborting.\n",
299 proxy_type_str(s->proxy), s->proxy->id);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200300 s->result = -1;
301 }
Willy Tarreau163c5322006-11-14 16:18:41 +0100302#ifdef CONFIG_HAP_CTTPROXY
303 if ((s->proxy->options & PR_O_TPXY_MASK) == PR_O_TPXY_ADDR) {
304 struct in_tproxy itp1, itp2;
305 memset(&itp1, 0, sizeof(itp1));
306
307 itp1.op = TPROXY_ASSIGN;
308 itp1.v.addr.faddr = s->tproxy_addr.sin_addr;
309 itp1.v.addr.fport = s->tproxy_addr.sin_port;
310
311 /* set connect flag on socket */
312 itp2.op = TPROXY_FLAGS;
313 itp2.v.flags = ITP_CONNECT | ITP_ONCE;
314
315 if (setsockopt(fd, SOL_IP, IP_TPROXY, &itp1, sizeof(itp1)) == -1 ||
316 setsockopt(fd, SOL_IP, IP_TPROXY, &itp2, sizeof(itp2)) == -1) {
Willy Tarreau2b5652f2006-12-31 17:46:05 +0100317 Alert("Cannot bind to tproxy source address before connect() for %s '%s'. Aborting.\n",
318 proxy_type_str(s->proxy), s->proxy->id);
Willy Tarreau163c5322006-11-14 16:18:41 +0100319 s->result = -1;
320 }
321 }
322#endif
Willy Tarreaubaaee002006-06-26 02:48:02 +0200323 }
324
325 if (!s->result) {
326 if ((connect(fd, (struct sockaddr *)&sa, sizeof(sa)) != -1) || (errno == EINPROGRESS)) {
327 /* OK, connection in progress or established */
328
329 //fprintf(stderr, "process_chk: 4\n");
330
331 s->curfd = fd; /* that's how we know a test is in progress ;-) */
332 fdtab[fd].owner = t;
Willy Tarreau54469402006-07-29 16:59:06 +0200333 fdtab[fd].cb[DIR_RD].f = &event_srv_chk_r;
334 fdtab[fd].cb[DIR_RD].b = NULL;
335 fdtab[fd].cb[DIR_WR].f = &event_srv_chk_w;
336 fdtab[fd].cb[DIR_WR].b = NULL;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200337 fdtab[fd].state = FD_STCONN; /* connection in progress */
Willy Tarreau2a429502006-10-15 14:52:29 +0200338 MY_FD_SET(fd, StaticWriteEvent); /* for connect status */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200339#ifdef DEBUG_FULL
Willy Tarreau2a429502006-10-15 14:52:29 +0200340 assert (!MY_FD_ISSET(fd, StaticReadEvent));
Willy Tarreaubaaee002006-06-26 02:48:02 +0200341#endif
342 fd_insert(fd);
343 /* FIXME: we allow up to <inter> for a connection to establish, but we should use another parameter */
344 tv_delayfrom(&t->expire, &now, s->inter);
345 task_queue(t); /* restore t to its place in the task list */
346 return tv_remain(&now, &t->expire);
347 }
348 else if (errno != EALREADY && errno != EISCONN && errno != EAGAIN) {
349 s->result = -1; /* a real error */
350 }
351 }
352 }
353 close(fd); /* socket creation error */
354 }
355
356 if (!s->result) { /* nothing done */
357 //fprintf(stderr, "process_chk: 6\n");
358 while (tv_cmp2_ms(&t->expire, &now) <= 0)
359 tv_delayfrom(&t->expire, &t->expire, s->inter);
360 goto new_chk; /* may be we should initialize a new check */
361 }
362
363 /* here, we have seen a failure */
364 if (s->health > s->rise) {
365 s->health--; /* still good */
366 s->failed_checks++;
367 }
368 else
369 set_server_down(s);
370
371 //fprintf(stderr, "process_chk: 7\n");
372 /* FIXME: we allow up to <inter> for a connection to establish, but we should use another parameter */
373 while (tv_cmp2_ms(&t->expire, &now) <= 0)
374 tv_delayfrom(&t->expire, &t->expire, s->inter);
375 goto new_chk;
376 }
377 else {
378 //fprintf(stderr, "process_chk: 8\n");
379 /* there was a test running */
380 if (s->result > 0) { /* good server detected */
381 //fprintf(stderr, "process_chk: 9\n");
382 s->health++; /* was bad, stays for a while */
383 if (s->health >= s->rise) {
384 s->state |= SRV_RUNNING;
385
386 if (s->health == s->rise) {
387 int xferred;
388
389 recount_servers(s->proxy);
390 recalc_server_map(s->proxy);
391
392 /* check if we can handle some connections queued at the proxy. We
393 * will take as many as we can handle.
394 */
395 for (xferred = 0; !s->maxconn || xferred < srv_dynamic_maxconn(s); xferred++) {
396 struct session *sess;
397 struct pendconn *p;
398
399 p = pendconn_from_px(s->proxy);
400 if (!p)
401 break;
402 p->sess->srv = s;
403 sess = p->sess;
404 pendconn_free(p);
405 task_wakeup(&rq, sess->task);
406 }
407
408 sprintf(trash,
409 "%sServer %s/%s is UP. %d active and %d backup servers online.%s"
410 " %d sessions requeued, %d total in queue.\n",
411 s->state & SRV_BACKUP ? "Backup " : "",
412 s->proxy->id, s->id, s->proxy->srv_act, s->proxy->srv_bck,
413 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
414 xferred, s->nbpend);
415
416 Warning("%s", trash);
417 send_log(s->proxy, LOG_NOTICE, "%s", trash);
418 }
419
420 s->health = s->rise + s->fall - 1; /* OK now */
421 }
422 s->curfd = -1; /* no check running anymore */
Willy Tarreau2a429502006-10-15 14:52:29 +0200423 //MY_FD_CLR(fd, StaticWriteEvent);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200424 fd_delete(fd);
425 while (tv_cmp2_ms(&t->expire, &now) <= 0)
426 tv_delayfrom(&t->expire, &t->expire, s->inter);
427 goto new_chk;
428 }
429 else if (s->result < 0 || tv_cmp2_ms(&t->expire, &now) <= 0) {
430 //fprintf(stderr, "process_chk: 10\n");
431 /* failure or timeout detected */
432 if (s->health > s->rise) {
433 s->health--; /* still good */
434 s->failed_checks++;
435 }
436 else
437 set_server_down(s);
438 s->curfd = -1;
Willy Tarreau2a429502006-10-15 14:52:29 +0200439 //MY_FD_CLR(fd, StaticWriteEvent);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200440 fd_delete(fd);
441 while (tv_cmp2_ms(&t->expire, &now) <= 0)
442 tv_delayfrom(&t->expire, &t->expire, s->inter);
443 goto new_chk;
444 }
445 /* if result is 0 and there's no timeout, we have to wait again */
446 }
447 //fprintf(stderr, "process_chk: 11\n");
448 s->result = 0;
449 task_queue(t); /* restore t to its place in the task list */
450 return tv_remain2(&now, &t->expire);
451}
452
453
454/*
455 * Local variables:
456 * c-indent-level: 8
457 * c-basic-offset: 8
458 * End:
459 */