blob: d326625be84ce08e775461bb1cfcd704491ffef2 [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
2 * Health-checks functions.
3 *
Willy Tarreau26c25062009-03-08 09:38:41 +01004 * Copyright 2000-2009 Willy Tarreau <w@1wt.eu>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02005 * Copyright 2007-2009 Krzysztof Piotr Oledzki <ole@ans.pl>
Willy Tarreaubaaee002006-06-26 02:48:02 +02006 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 *
12 */
13
Willy Tarreaub8816082008-01-18 12:18:15 +010014#include <assert.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020015#include <ctype.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020016#include <errno.h>
17#include <fcntl.h>
18#include <stdio.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020019#include <stdlib.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020020#include <string.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020021#include <time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020022#include <unistd.h>
23#include <sys/socket.h>
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +040024#include <sys/types.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020025#include <netinet/in.h>
Willy Tarreau1274bc42009-07-15 07:16:31 +020026#include <netinet/tcp.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020027#include <arpa/inet.h>
28
Willy Tarreau2dd0d472006-06-29 17:53:05 +020029#include <common/compat.h>
30#include <common/config.h>
31#include <common/mini-clist.h>
Willy Tarreau83749182007-04-15 20:56:27 +020032#include <common/standard.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020033#include <common/time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020034
35#include <types/global.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020036
37#include <proto/backend.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020038#include <proto/checks.h>
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +010039#include <proto/buffers.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020040#include <proto/fd.h>
41#include <proto/log.h>
42#include <proto/queue.h>
Willy Tarreauc6f4ce82009-06-10 11:09:37 +020043#include <proto/port_range.h>
Willy Tarreau3d300592007-03-18 18:34:41 +010044#include <proto/proto_http.h>
Willy Tarreaue8c66af2008-01-13 18:40:14 +010045#include <proto/proto_tcp.h>
Willy Tarreau2b5652f2006-12-31 17:46:05 +010046#include <proto/proxy.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020047#include <proto/server.h>
48#include <proto/task.h>
49
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020050const struct check_status check_statuses[HCHK_STATUS_SIZE] = {
51 [HCHK_STATUS_UNKNOWN] = { SRV_CHK_UNKNOWN, "UNK", "Unknown" },
52 [HCHK_STATUS_INI] = { SRV_CHK_UNKNOWN, "INI", "Initializing" },
53 [HCHK_STATUS_START] = { /* SPECIAL STATUS*/ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020054
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010055 [HCHK_STATUS_HANA] = { SRV_CHK_ERROR, "HANA", "Health analyze" },
56
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020057 [HCHK_STATUS_SOCKERR] = { SRV_CHK_ERROR, "SOCKERR", "Socket error" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020058
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020059 [HCHK_STATUS_L4OK] = { SRV_CHK_RUNNING, "L4OK", "Layer4 check passed" },
60 [HCHK_STATUS_L4TOUT] = { SRV_CHK_ERROR, "L4TOUT", "Layer4 timeout" },
61 [HCHK_STATUS_L4CON] = { SRV_CHK_ERROR, "L4CON", "Layer4 connection problem" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020062
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020063 [HCHK_STATUS_L6OK] = { SRV_CHK_RUNNING, "L6OK", "Layer6 check passed" },
64 [HCHK_STATUS_L6TOUT] = { SRV_CHK_ERROR, "L6TOUT", "Layer6 timeout" },
65 [HCHK_STATUS_L6RSP] = { SRV_CHK_ERROR, "L6RSP", "Layer6 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020066
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020067 [HCHK_STATUS_L7TOUT] = { SRV_CHK_ERROR, "L7TOUT", "Layer7 timeout" },
68 [HCHK_STATUS_L7RSP] = { SRV_CHK_ERROR, "L7RSP", "Layer7 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020069
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020070 [HCHK_STATUS_L57DATA] = { /* DUMMY STATUS */ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020071
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020072 [HCHK_STATUS_L7OKD] = { SRV_CHK_RUNNING, "L7OK", "Layer7 check passed" },
73 [HCHK_STATUS_L7OKCD] = { SRV_CHK_RUNNING | SRV_CHK_DISABLE, "L7OKC", "Layer7 check conditionally passed" },
74 [HCHK_STATUS_L7STS] = { SRV_CHK_ERROR, "L7STS", "Layer7 wrong status" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020075};
76
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010077const struct analyze_status analyze_statuses[HANA_STATUS_SIZE] = { /* 0: ignore, 1: error, 2: OK */
78 [HANA_STATUS_UNKNOWN] = { "Unknown", { 0, 0 }},
79
80 [HANA_STATUS_L4_OK] = { "L4 successful connection", { 2, 0 }},
81 [HANA_STATUS_L4_ERR] = { "L4 unsuccessful connection", { 1, 1 }},
82
83 [HANA_STATUS_HTTP_OK] = { "Correct http response", { 0, 2 }},
84 [HANA_STATUS_HTTP_STS] = { "Wrong http response", { 0, 1 }},
85 [HANA_STATUS_HTTP_HDRRSP] = { "Invalid http response (headers)", { 0, 1 }},
86 [HANA_STATUS_HTTP_RSP] = { "Invalid http response", { 0, 1 }},
87
88 [HANA_STATUS_HTTP_READ_ERROR] = { "Read error (http)", { 0, 1 }},
89 [HANA_STATUS_HTTP_READ_TIMEOUT] = { "Read timeout (http)", { 0, 1 }},
90 [HANA_STATUS_HTTP_BROKEN_PIPE] = { "Close from server (http)", { 0, 1 }},
91};
92
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020093/*
94 * Convert check_status code to description
95 */
96const char *get_check_status_description(short check_status) {
97
98 const char *desc;
99
100 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200101 desc = check_statuses[check_status].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200102 else
103 desc = NULL;
104
105 if (desc && *desc)
106 return desc;
107 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200108 return check_statuses[HCHK_STATUS_UNKNOWN].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200109}
110
111/*
112 * Convert check_status code to short info
113 */
114const char *get_check_status_info(short check_status) {
115
116 const char *info;
117
118 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200119 info = check_statuses[check_status].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200120 else
121 info = NULL;
122
123 if (info && *info)
124 return info;
125 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200126 return check_statuses[HCHK_STATUS_UNKNOWN].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200127}
128
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100129const char *get_analyze_status(short analyze_status) {
130
131 const char *desc;
132
133 if (analyze_status < HANA_STATUS_SIZE)
134 desc = analyze_statuses[analyze_status].desc;
135 else
136 desc = NULL;
137
138 if (desc && *desc)
139 return desc;
140 else
141 return analyze_statuses[HANA_STATUS_UNKNOWN].desc;
142}
143
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200144#define SSP_O_HCHK 0x0002
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200145
146static void server_status_printf(struct chunk *msg, struct server *s, unsigned options, int xferred) {
147
Krzysztof Piotr Oledzki5f5b7d22010-01-11 11:13:39 +0100148 if (s->tracked)
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200149 chunk_printf(msg, " via %s/%s",
150 s->tracked->proxy->id, s->tracked->id);
151
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200152 if (options & SSP_O_HCHK) {
153 chunk_printf(msg, ", reason: %s", get_check_status_description(s->check_status));
154
155 if (s->check_status >= HCHK_STATUS_L57DATA)
156 chunk_printf(msg, ", code: %d", s->check_code);
157
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200158 if (*s->check_desc) {
159 struct chunk src;
160
161 chunk_printf(msg, ", info: \"");
162
163 chunk_initlen(&src, s->check_desc, 0, strlen(s->check_desc));
164 chunk_asciiencode(msg, &src, '"');
165
166 chunk_printf(msg, "\"");
167 }
168
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100169 if (s->check_duration >= 0)
170 chunk_printf(msg, ", check duration: %ldms", s->check_duration);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200171 }
172
Krzysztof Piotr Oledzki5f5b7d22010-01-11 11:13:39 +0100173 if (xferred > 0) {
Krzysztof Piotr Oledzkib16a6072010-01-10 21:12:58 +0100174 if (!(s->state & SRV_RUNNING))
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200175 chunk_printf(msg, ". %d active and %d backup servers left.%s"
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100176 " %d sessions active, %d requeued, %d remaining in queue",
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200177 s->proxy->srv_act, s->proxy->srv_bck,
178 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
179 s->cur_sess, xferred, s->nbpend);
180 else
181 chunk_printf(msg, ". %d active and %d backup servers online.%s"
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100182 " %d sessions requeued, %d total in queue",
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200183 s->proxy->srv_act, s->proxy->srv_bck,
184 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
185 xferred, s->nbpend);
186 }
187}
188
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200189/*
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200190 * Set s->check_status, update s->check_duration and fill s->result with
191 * an adequate SRV_CHK_* value.
192 *
193 * Show information in logs about failed health check if server is UP
194 * or succeeded health checks if server is DOWN.
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200195 */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200196static void set_server_check_status(struct server *s, short status, char *desc) {
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200197
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200198 struct chunk msg;
199
200 if (status == HCHK_STATUS_START) {
201 s->result = SRV_CHK_UNKNOWN; /* no result yet */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200202 s->check_desc[0] = '\0';
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200203 s->check_start = now;
204 return;
205 }
206
207 if (!s->check_status)
208 return;
209
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200210 if (desc && *desc) {
211 strncpy(s->check_desc, desc, HCHK_DESC_LEN-1);
212 s->check_desc[HCHK_DESC_LEN-1] = '\0';
213 } else
214 s->check_desc[0] = '\0';
215
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200216 s->check_status = status;
217 if (check_statuses[status].result)
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100218 s->result = check_statuses[status].result;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200219
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100220 if (status == HCHK_STATUS_HANA)
221 s->check_duration = -1;
222 else if (!tv_iszero(&s->check_start)) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200223 /* set_server_check_status() may be called more than once */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200224 s->check_duration = tv_ms_elapsed(&s->check_start, &now);
225 tv_zero(&s->check_start);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200226 }
227
228 if (s->proxy->options2 & PR_O2_LOGHCHKS &&
229 (((s->health != 0) && (s->result & SRV_CHK_ERROR)) ||
230 ((s->health != s->rise + s->fall - 1) && (s->result & SRV_CHK_RUNNING)) ||
231 ((s->state & SRV_GOINGDOWN) && !(s->result & SRV_CHK_DISABLE)) ||
232 (!(s->state & SRV_GOINGDOWN) && (s->result & SRV_CHK_DISABLE)))) {
233
234 int health, rise, fall, state;
235
236 chunk_init(&msg, trash, sizeof(trash));
237
238 /* FIXME begin: calculate local version of the health/rise/fall/state */
239 health = s->health;
240 rise = s->rise;
241 fall = s->fall;
242 state = s->state;
243
244 if (s->result & SRV_CHK_ERROR) {
245 if (health > rise) {
246 health--; /* still good */
247 } else {
248 if (health == rise)
249 state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
250
251 health = 0;
252 }
253 }
254
255 if (s->result & SRV_CHK_RUNNING) {
256 if (health < rise + fall - 1) {
257 health++; /* was bad, stays for a while */
258
259 if (health == rise)
260 state |= SRV_RUNNING;
261
262 if (health >= rise)
263 health = rise + fall - 1; /* OK now */
264 }
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100265
266 /* clear consecutive_errors if observing is enabled */
267 if (s->onerror)
268 s->consecutive_errors = 0;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200269 }
270 /* FIXME end: calculate local version of the health/rise/fall/state */
271
272 chunk_printf(&msg,
273 "Health check for %sserver %s/%s %s%s",
274 s->state & SRV_BACKUP ? "backup " : "",
275 s->proxy->id, s->id,
276 (s->result & SRV_CHK_DISABLE)?"conditionally ":"",
277 (s->result & SRV_CHK_RUNNING)?"succeeded":"failed");
278
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200279 server_status_printf(&msg, s, SSP_O_HCHK, -1);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200280
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100281 chunk_printf(&msg, ", status: %d/%d %s",
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200282 (state & SRV_RUNNING) ? (health - rise + 1) : (health),
283 (state & SRV_RUNNING) ? (fall) : (rise),
284 (state & SRV_RUNNING)?"UP":"DOWN");
285
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100286 Warning("%s.\n", trash);
287 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200288 }
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200289}
290
Willy Tarreau48494c02007-11-30 10:41:39 +0100291/* sends a log message when a backend goes down, and also sets last
292 * change date.
293 */
294static void set_backend_down(struct proxy *be)
295{
296 be->last_change = now.tv_sec;
297 be->down_trans++;
298
299 Alert("%s '%s' has no server available!\n", proxy_type_str(be), be->id);
300 send_log(be, LOG_EMERG, "%s %s has no server available!\n", proxy_type_str(be), be->id);
301}
302
303/* Redistribute pending connections when a server goes down. The number of
304 * connections redistributed is returned.
305 */
306static int redistribute_pending(struct server *s)
307{
308 struct pendconn *pc, *pc_bck, *pc_end;
309 int xferred = 0;
310
311 FOREACH_ITEM_SAFE(pc, pc_bck, &s->pendconns, pc_end, struct pendconn *, list) {
312 struct session *sess = pc->sess;
Willy Tarreau4de91492010-01-22 19:10:05 +0100313 if ((sess->be->options & (PR_O_REDISP|PR_O_PERSIST)) == PR_O_REDISP &&
314 !(sess->flags & SN_FORCE_PRST)) {
Willy Tarreau48494c02007-11-30 10:41:39 +0100315 /* The REDISP option was specified. We will ignore
316 * cookie and force to balance or use the dispatcher.
317 */
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100318
Krzysztof Piotr Oledzki5a329cf2008-02-22 03:50:19 +0100319 /* it's left to the dispatcher to choose a server */
Willy Tarreau48494c02007-11-30 10:41:39 +0100320 sess->flags &= ~(SN_DIRECT | SN_ASSIGNED | SN_ADDR_SET);
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100321
Willy Tarreau48494c02007-11-30 10:41:39 +0100322 pendconn_free(pc);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200323 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100324 xferred++;
325 }
326 }
327 return xferred;
328}
329
330/* Check for pending connections at the backend, and assign some of them to
331 * the server coming up. The server's weight is checked before being assigned
332 * connections it may not be able to handle. The total number of transferred
333 * connections is returned.
334 */
335static int check_for_pending(struct server *s)
336{
337 int xferred;
338
339 if (!s->eweight)
340 return 0;
341
342 for (xferred = 0; !s->maxconn || xferred < srv_dynamic_maxconn(s); xferred++) {
343 struct session *sess;
344 struct pendconn *p;
345
346 p = pendconn_from_px(s->proxy);
347 if (!p)
348 break;
349 p->sess->srv = s;
350 sess = p->sess;
351 pendconn_free(p);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200352 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100353 }
354 return xferred;
355}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200356
357/* Sets server <s> down, notifies by all available means, recounts the
358 * remaining servers on the proxy and transfers queued sessions whenever
Willy Tarreau5af3a692007-07-24 23:32:33 +0200359 * possible to other servers. It automatically recomputes the number of
360 * servers, but not the map.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200361 */
Cyril Bontécd19e512010-01-31 22:34:03 +0100362void set_server_down(struct server *s)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200363{
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100364 struct server *srv;
365 struct chunk msg;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200366 int xferred;
367
Cyril Bontécd19e512010-01-31 22:34:03 +0100368 if (s->state & SRV_MAINTAIN) {
369 s->health = s->rise;
370 }
371
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100372 if (s->health == s->rise || s->tracked) {
Willy Tarreau48494c02007-11-30 10:41:39 +0100373 int srv_was_paused = s->state & SRV_GOINGDOWN;
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200374
375 s->last_change = now.tv_sec;
Willy Tarreau48494c02007-11-30 10:41:39 +0100376 s->state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
Willy Tarreaub625a082007-11-26 01:15:43 +0100377 s->proxy->lbprm.set_server_status_down(s);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200378
379 /* we might have sessions queued on this server and waiting for
380 * a connection. Those which are redispatchable will be queued
381 * to another server or to the proxy itself.
382 */
Willy Tarreau48494c02007-11-30 10:41:39 +0100383 xferred = redistribute_pending(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100384
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200385 chunk_init(&msg, trash, sizeof(trash));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100386
Cyril Bontécd19e512010-01-31 22:34:03 +0100387 if (s->state & SRV_MAINTAIN) {
388 chunk_printf(&msg,
389 "%sServer %s/%s is DOWN for maintenance", s->state & SRV_BACKUP ? "Backup " : "",
390 s->proxy->id, s->id);
391 } else {
392 chunk_printf(&msg,
393 "%sServer %s/%s is DOWN", s->state & SRV_BACKUP ? "Backup " : "",
394 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100395
Cyril Bontécd19e512010-01-31 22:34:03 +0100396 server_status_printf(&msg, s,
397 ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
398 xferred);
399 }
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100400 Warning("%s.\n", trash);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200401
Willy Tarreau48494c02007-11-30 10:41:39 +0100402 /* we don't send an alert if the server was previously paused */
403 if (srv_was_paused)
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100404 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash);
Willy Tarreau48494c02007-11-30 10:41:39 +0100405 else
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100406 send_log(s->proxy, LOG_ALERT, "%s.\n", trash);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200407
Willy Tarreau48494c02007-11-30 10:41:39 +0100408 if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0)
409 set_backend_down(s->proxy);
410
Krzysztof Piotr Oledzki052d4fd2009-10-04 14:52:57 +0200411 s->counters.down_trans++;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100412
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100413 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100414 for(srv = s->tracknext; srv; srv = srv->tracknext)
Cyril Bontécd19e512010-01-31 22:34:03 +0100415 if (! (srv->state & SRV_MAINTAIN))
416 /* Only notify tracking servers that are not already in maintenance. */
417 set_server_down(srv);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200418 }
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100419
Willy Tarreaubaaee002006-06-26 02:48:02 +0200420 s->health = 0; /* failure */
421}
422
Cyril Bontécd19e512010-01-31 22:34:03 +0100423void set_server_up(struct server *s) {
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100424
425 struct server *srv;
426 struct chunk msg;
427 int xferred;
428
Cyril Bontécd19e512010-01-31 22:34:03 +0100429 if (s->state & SRV_MAINTAIN) {
430 s->health = s->rise;
431 }
432
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100433 if (s->health == s->rise || s->tracked) {
434 if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0) {
435 if (s->proxy->last_change < now.tv_sec) // ignore negative times
436 s->proxy->down_time += now.tv_sec - s->proxy->last_change;
437 s->proxy->last_change = now.tv_sec;
438 }
439
440 if (s->last_change < now.tv_sec) // ignore negative times
441 s->down_time += now.tv_sec - s->last_change;
442
443 s->last_change = now.tv_sec;
444 s->state |= SRV_RUNNING;
445
446 if (s->slowstart > 0) {
447 s->state |= SRV_WARMINGUP;
448 if (s->proxy->lbprm.algo & BE_LB_PROP_DYN) {
449 /* For dynamic algorithms, start at the first step of the weight,
450 * without multiplying by BE_WEIGHT_SCALE.
451 */
452 s->eweight = s->uweight;
453 if (s->proxy->lbprm.update_server_eweight)
454 s->proxy->lbprm.update_server_eweight(s);
455 }
456 }
457 s->proxy->lbprm.set_server_status_up(s);
458
459 /* check if we can handle some connections queued at the proxy. We
460 * will take as many as we can handle.
461 */
462 xferred = check_for_pending(s);
463
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200464 chunk_init(&msg, trash, sizeof(trash));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100465
Cyril Bontécd19e512010-01-31 22:34:03 +0100466 if (s->state & SRV_MAINTAIN) {
467 chunk_printf(&msg,
468 "%sServer %s/%s is UP (leaving maintenance)", s->state & SRV_BACKUP ? "Backup " : "",
469 s->proxy->id, s->id);
470 } else {
471 chunk_printf(&msg,
472 "%sServer %s/%s is UP", s->state & SRV_BACKUP ? "Backup " : "",
473 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100474
Cyril Bontécd19e512010-01-31 22:34:03 +0100475 server_status_printf(&msg, s,
476 ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
477 xferred);
478 }
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100479
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100480 Warning("%s.\n", trash);
481 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100482
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100483 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100484 for(srv = s->tracknext; srv; srv = srv->tracknext)
Cyril Bontécd19e512010-01-31 22:34:03 +0100485 if (! (srv->state & SRV_MAINTAIN))
486 /* Only notify tracking servers if they're not in maintenance. */
487 set_server_up(srv);
488
489 s->state &= ~SRV_MAINTAIN;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100490 }
491
492 if (s->health >= s->rise)
493 s->health = s->rise + s->fall - 1; /* OK now */
494
495}
496
497static void set_server_disabled(struct server *s) {
498
499 struct server *srv;
500 struct chunk msg;
501 int xferred;
502
503 s->state |= SRV_GOINGDOWN;
504 s->proxy->lbprm.set_server_status_down(s);
505
506 /* we might have sessions queued on this server and waiting for
507 * a connection. Those which are redispatchable will be queued
508 * to another server or to the proxy itself.
509 */
510 xferred = redistribute_pending(s);
511
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200512 chunk_init(&msg, trash, sizeof(trash));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100513
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200514 chunk_printf(&msg,
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100515 "Load-balancing on %sServer %s/%s is disabled",
516 s->state & SRV_BACKUP ? "Backup " : "",
517 s->proxy->id, s->id);
518
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200519 server_status_printf(&msg, s,
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200520 ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
Krzysztof Piotr Oledzkib16a6072010-01-10 21:12:58 +0100521 xferred);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100522
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100523 Warning("%s.\n", trash);
524 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100525
526 if (!s->proxy->srv_bck && !s->proxy->srv_act)
527 set_backend_down(s->proxy);
528
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100529 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100530 for(srv = s->tracknext; srv; srv = srv->tracknext)
531 set_server_disabled(srv);
532}
533
534static void set_server_enabled(struct server *s) {
535
536 struct server *srv;
537 struct chunk msg;
538 int xferred;
539
540 s->state &= ~SRV_GOINGDOWN;
541 s->proxy->lbprm.set_server_status_up(s);
542
543 /* check if we can handle some connections queued at the proxy. We
544 * will take as many as we can handle.
545 */
546 xferred = check_for_pending(s);
547
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200548 chunk_init(&msg, trash, sizeof(trash));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100549
Krzysztof Piotr Oledzki78abe612009-09-27 13:23:20 +0200550 chunk_printf(&msg,
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100551 "Load-balancing on %sServer %s/%s is enabled again",
552 s->state & SRV_BACKUP ? "Backup " : "",
553 s->proxy->id, s->id);
554
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200555 server_status_printf(&msg, s,
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200556 ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
Krzysztof Piotr Oledzkib16a6072010-01-10 21:12:58 +0100557 xferred);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100558
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100559 Warning("%s.\n", trash);
560 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100561
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100562 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100563 for(srv = s->tracknext; srv; srv = srv->tracknext)
564 set_server_enabled(srv);
565}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200566
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100567void health_adjust(struct server *s, short status) {
568
569 int failed;
570 int expire;
571
572 /* return now if observing nor health check is not enabled */
573 if (!s->observe || !s->check)
574 return;
575
576 if (s->observe >= HANA_OBS_SIZE)
577 return;
578
579 if (status >= HCHK_STATUS_SIZE || !analyze_statuses[status].desc)
580 return;
581
582 switch (analyze_statuses[status].lr[s->observe - 1]) {
583 case 1:
584 failed = 1;
585 break;
586
587 case 2:
588 failed = 0;
589 break;
590
591 default:
592 return;
593 }
594
595 if (!failed) {
596 /* good: clear consecutive_errors */
597 s->consecutive_errors = 0;
598 return;
599 }
600
601 s->consecutive_errors++;
602
603 if (s->consecutive_errors < s->consecutive_errors_limit)
604 return;
605
606 sprintf(trash, "Detected %d consecutive errors, last one was: %s",
607 s->consecutive_errors, get_analyze_status(status));
608
609 switch (s->onerror) {
610 case HANA_ONERR_FASTINTER:
611 /* force fastinter - nothing to do here as all modes force it */
612 break;
613
614 case HANA_ONERR_SUDDTH:
615 /* simulate a pre-fatal failed health check */
616 if (s->health > s->rise)
617 s->health = s->rise + 1;
618
619 /* no break - fall through */
620
621 case HANA_ONERR_FAILCHK:
622 /* simulate a failed health check */
623 set_server_check_status(s, HCHK_STATUS_HANA, trash);
624
625 if (s->health > s->rise) {
626 s->health--; /* still good */
627 s->counters.failed_checks++;
628 }
629 else
630 set_server_down(s);
631
632 break;
633
634 case HANA_ONERR_MARKDWN:
635 /* mark server down */
636 s->health = s->rise;
637 set_server_check_status(s, HCHK_STATUS_HANA, trash);
638 set_server_down(s);
639
640 break;
641
642 default:
643 /* write a warning? */
644 break;
645 }
646
647 s->consecutive_errors = 0;
648 s->counters.failed_hana++;
649
650 if (s->fastinter) {
651 expire = tick_add(now_ms, MS_TO_TICKS(s->fastinter));
652 if (s->check->expire > expire)
653 s->check->expire = expire;
654 }
655}
656
Willy Tarreauef781042010-01-27 11:53:01 +0100657static int httpchk_build_status_header(struct server *s, char *buffer)
658{
659 int sv_state;
660 int ratio;
661 int hlen = 0;
662 const char *srv_hlt_st[7] = { "DOWN", "DOWN %d/%d",
663 "UP %d/%d", "UP",
664 "NOLB %d/%d", "NOLB",
665 "no check" };
666
667 memcpy(buffer + hlen, "X-Haproxy-Server-State: ", 24);
668 hlen += 24;
669
670 if (!(s->state & SRV_CHECKED))
671 sv_state = 6; /* should obviously never happen */
672 else if (s->state & SRV_RUNNING) {
673 if (s->health == s->rise + s->fall - 1)
674 sv_state = 3; /* UP */
675 else
676 sv_state = 2; /* going down */
677
678 if (s->state & SRV_GOINGDOWN)
679 sv_state += 2;
680 } else {
681 if (s->health)
682 sv_state = 1; /* going up */
683 else
684 sv_state = 0; /* DOWN */
685 }
686
687 hlen += sprintf(buffer + hlen,
688 srv_hlt_st[sv_state],
689 (s->state & SRV_RUNNING) ? (s->health - s->rise + 1) : (s->health),
690 (s->state & SRV_RUNNING) ? (s->fall) : (s->rise));
691
692 hlen += sprintf(buffer + hlen, "; name=%s/%s; node=%s; weight=%d/%d; scur=%d/%d; qcur=%d",
693 s->proxy->id, s->id,
694 global.node,
695 (s->eweight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv,
696 (s->proxy->lbprm.tot_weight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv,
697 s->cur_sess, s->proxy->beconn - s->proxy->nbpend,
698 s->nbpend);
699
700 if ((s->state & SRV_WARMINGUP) &&
701 now.tv_sec < s->last_change + s->slowstart &&
702 now.tv_sec >= s->last_change) {
703 ratio = MAX(1, 100 * (now.tv_sec - s->last_change) / s->slowstart);
704 hlen += sprintf(buffer + hlen, "; throttle=%d%%", ratio);
705 }
706
707 buffer[hlen++] = '\r';
708 buffer[hlen++] = '\n';
709
710 return hlen;
711}
712
Willy Tarreaubaaee002006-06-26 02:48:02 +0200713/*
714 * This function is used only for server health-checks. It handles
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200715 * the connection acknowledgement. If the proxy requires L7 health-checks,
716 * it sends the request. In other cases, it calls set_server_check_status()
717 * to set s->check_status, s->check_duration and s->result.
Willy Tarreau83749182007-04-15 20:56:27 +0200718 * The function itself returns 0 if it needs some polling before being called
719 * again, otherwise 1.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200720 */
Willy Tarreau83749182007-04-15 20:56:27 +0200721static int event_srv_chk_w(int fd)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200722{
Willy Tarreau6996e152007-04-30 14:37:43 +0200723 __label__ out_wakeup, out_nowake, out_poll, out_error;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200724 struct task *t = fdtab[fd].owner;
725 struct server *s = t->context;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200726
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +0100727 //fprintf(stderr, "event_srv_chk_w, state=%ld\n", unlikely(fdtab[fd].state));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200728 if (unlikely(fdtab[fd].state == FD_STERROR || (fdtab[fd].ev & FD_POLL_ERR))) {
Krzysztof Piotr Oledzki6492db52010-01-02 22:03:01 +0100729 int skerr, err = errno;
730 socklen_t lskerr = sizeof(skerr);
731
732 if (!getsockopt(fd, SOL_SOCKET, SO_ERROR, &skerr, &lskerr) && skerr)
733 err = skerr;
734
735 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(err));
Willy Tarreau6996e152007-04-30 14:37:43 +0200736 goto out_error;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200737 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200738
739 /* here, we know that the connection is established */
Willy Tarreau83749182007-04-15 20:56:27 +0200740
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100741 if (!(s->result & SRV_CHK_ERROR)) {
Willy Tarreaubaaee002006-06-26 02:48:02 +0200742 /* we don't want to mark 'UP' a server on which we detected an error earlier */
Willy Tarreauf3c69202006-07-09 16:42:34 +0200743 if ((s->proxy->options & PR_O_HTTP_CHK) ||
Hervé COMMOWICK698ae002010-01-12 09:25:13 +0100744 (s->proxy->options & PR_O_SMTP_CHK) ||
Willy Tarreau07a54902010-03-29 18:33:29 +0200745 (s->proxy->options2 & PR_O2_SSL3_CHK) ||
Hervé COMMOWICK698ae002010-01-12 09:25:13 +0100746 (s->proxy->options2 & PR_O2_MYSQL_CHK)) {
Willy Tarreaubaaee002006-06-26 02:48:02 +0200747 int ret;
Willy Tarreaue9d87882010-01-27 11:28:42 +0100748 const char *check_req = s->proxy->check_req;
749 int check_len = s->proxy->check_len;
750
Willy Tarreauf3c69202006-07-09 16:42:34 +0200751 /* we want to check if this host replies to HTTP or SSLv3 requests
Willy Tarreaubaaee002006-06-26 02:48:02 +0200752 * so we'll send the request, and won't wake the checker up now.
753 */
Willy Tarreauf3c69202006-07-09 16:42:34 +0200754
Willy Tarreau07a54902010-03-29 18:33:29 +0200755 if (s->proxy->options2 & PR_O2_SSL3_CHK) {
Willy Tarreauf3c69202006-07-09 16:42:34 +0200756 /* SSL requires that we put Unix time in the request */
Willy Tarreaub7f694f2008-06-22 17:18:02 +0200757 int gmt_time = htonl(date.tv_sec);
Willy Tarreauf3c69202006-07-09 16:42:34 +0200758 memcpy(s->proxy->check_req + 11, &gmt_time, 4);
759 }
Willy Tarreaue9d87882010-01-27 11:28:42 +0100760 else if (s->proxy->options & PR_O_HTTP_CHK) {
761 memcpy(trash, check_req, check_len);
Willy Tarreauef781042010-01-27 11:53:01 +0100762
763 if (s->proxy->options2 & PR_O2_CHK_SNDST)
764 check_len += httpchk_build_status_header(s, trash + check_len);
765
Willy Tarreaue9d87882010-01-27 11:28:42 +0100766 trash[check_len++] = '\r';
767 trash[check_len++] = '\n';
768 trash[check_len] = '\0';
769 check_req = trash;
770 }
Willy Tarreauf3c69202006-07-09 16:42:34 +0200771
Willy Tarreaue9d87882010-01-27 11:28:42 +0100772 ret = send(fd, check_req, check_len, MSG_DONTWAIT | MSG_NOSIGNAL);
773 if (ret == check_len) {
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +0100774 /* we allow up to <timeout.check> if nonzero for a responce */
Willy Tarreau7cd9d942008-12-21 13:00:41 +0100775 if (s->proxy->timeout.check)
776 t->expire = tick_add_ifset(now_ms, s->proxy->timeout.check);
Willy Tarreauf161a342007-04-08 16:59:42 +0200777 EV_FD_SET(fd, DIR_RD); /* prepare for reading reply */
Willy Tarreau83749182007-04-15 20:56:27 +0200778 goto out_nowake;
779 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200780 else if (ret == 0 || errno == EAGAIN)
781 goto out_poll;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200782 else {
783 switch (errno) {
784 case ECONNREFUSED:
785 case ENETUNREACH:
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200786 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(errno));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200787 break;
788
789 default:
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200790 set_server_check_status(s, HCHK_STATUS_SOCKERR, strerror(errno));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200791 }
792
Willy Tarreau6996e152007-04-30 14:37:43 +0200793 goto out_error;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200794 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200795 }
796 else {
Willy Tarreau6996e152007-04-30 14:37:43 +0200797 /* We have no data to send to check the connection, and
798 * getsockopt() will not inform us whether the connection
799 * is still pending. So we'll reuse connect() to check the
800 * state of the socket. This has the advantage of givig us
801 * the following info :
802 * - error
803 * - connecting (EALREADY, EINPROGRESS)
804 * - connected (EISCONN, 0)
805 */
806
807 struct sockaddr_in sa;
808
809 sa = (s->check_addr.sin_addr.s_addr) ? s->check_addr : s->addr;
810 sa.sin_port = htons(s->check_port);
811
812 if (connect(fd, (struct sockaddr *)&sa, sizeof(sa)) == 0)
813 errno = 0;
814
815 if (errno == EALREADY || errno == EINPROGRESS)
816 goto out_poll;
817
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200818 if (errno && errno != EISCONN) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200819 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(errno));
Willy Tarreau6996e152007-04-30 14:37:43 +0200820 goto out_error;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200821 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200822
Willy Tarreaubaaee002006-06-26 02:48:02 +0200823 /* good TCP connection is enough */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200824 set_server_check_status(s, HCHK_STATUS_L4OK, NULL);
Willy Tarreau6996e152007-04-30 14:37:43 +0200825 goto out_wakeup;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200826 }
827 }
Willy Tarreau83749182007-04-15 20:56:27 +0200828 out_wakeup:
Willy Tarreaufdccded2008-08-29 18:19:04 +0200829 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreau83749182007-04-15 20:56:27 +0200830 out_nowake:
831 EV_FD_CLR(fd, DIR_WR); /* nothing more to write */
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100832 fdtab[fd].ev &= ~FD_POLL_OUT;
Willy Tarreau83749182007-04-15 20:56:27 +0200833 return 1;
Willy Tarreau6996e152007-04-30 14:37:43 +0200834 out_poll:
835 /* The connection is still pending. We'll have to poll it
836 * before attempting to go further. */
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100837 fdtab[fd].ev &= ~FD_POLL_OUT;
Willy Tarreau6996e152007-04-30 14:37:43 +0200838 return 0;
839 out_error:
Willy Tarreau6996e152007-04-30 14:37:43 +0200840 fdtab[fd].state = FD_STERROR;
841 goto out_wakeup;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200842}
843
844
845/*
Willy Tarreauf3c69202006-07-09 16:42:34 +0200846 * This function is used only for server health-checks. It handles the server's
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200847 * reply to an HTTP request or SSL HELLO. It calls set_server_check_status() to
848 * update s->check_status, s->check_duration and s->result.
849
850 * The set_server_check_status function is called with HCHK_STATUS_L7OKD if
851 * an HTTP server replies HTTP 2xx or 3xx (valid responses), if an SMTP server
852 * returns 2xx, HCHK_STATUS_L6OK if an SSL server returns at least 5 bytes in
853 * response to an SSL HELLO (the principle is that this is enough to
854 * distinguish between an SSL server and a pure TCP relay). All other cases will
855 * call it with a proper error status like HCHK_STATUS_L7STS, HCHK_STATUS_L6RSP,
856 * etc.
857 *
858 * The function returns 0 if it needs to be called again after some polling,
859 * otherwise non-zero..
Willy Tarreaubaaee002006-06-26 02:48:02 +0200860 */
Willy Tarreau83749182007-04-15 20:56:27 +0200861static int event_srv_chk_r(int fd)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200862{
Willy Tarreau83749182007-04-15 20:56:27 +0200863 __label__ out_wakeup;
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100864 int len;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200865 struct task *t = fdtab[fd].owner;
866 struct server *s = t->context;
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200867 char *desc;
Willy Tarreau03938182010-03-17 21:52:07 +0100868 int done;
Willy Tarreau83749182007-04-15 20:56:27 +0200869
Willy Tarreau659d7bc2010-03-16 21:14:41 +0100870 if (unlikely((s->result & SRV_CHK_ERROR) || (fdtab[fd].state == FD_STERROR))) {
Willy Tarreau83749182007-04-15 20:56:27 +0200871 /* in case of TCP only, this tells us if the connection failed */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200872 if (!(s->result & SRV_CHK_ERROR))
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200873 set_server_check_status(s, HCHK_STATUS_SOCKERR, NULL);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200874
Willy Tarreau83749182007-04-15 20:56:27 +0200875 goto out_wakeup;
876 }
877
Willy Tarreau83749182007-04-15 20:56:27 +0200878 /* Warning! Linux returns EAGAIN on SO_ERROR if data are still available
879 * but the connection was closed on the remote end. Fortunately, recv still
880 * works correctly and we don't need to do the getsockopt() on linux.
881 */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000882
883 /* Set buffer to point to the end of the data already read, and check
884 * that there is free space remaining. If the buffer is full, proceed
885 * with running the checks without attempting another socket read.
886 */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000887
Willy Tarreau03938182010-03-17 21:52:07 +0100888 done = 0;
Willy Tarreau2c7ace02010-03-16 20:32:04 +0100889 for (len = 0; s->check_data_len < BUFSIZE; s->check_data_len += len) {
890 len = recv(fd, s->check_data + s->check_data_len, BUFSIZE - s->check_data_len, 0);
891 if (len <= 0)
892 break;
893 }
Nick Chalk57b1bf72010-03-16 15:50:46 +0000894
Willy Tarreau03938182010-03-17 21:52:07 +0100895 if (len == 0)
896 done = 1; /* connection hangup received */
897 else if (len < 0 && errno != EAGAIN) {
Willy Tarreauc1a07962010-03-16 20:55:43 +0100898 /* Report network errors only if we got no other data. Otherwise
899 * we'll let the upper layers decide whether the response is OK
900 * or not. It is very common that an RST sent by the server is
901 * reported as an error just after the last data chunk.
902 */
Willy Tarreau03938182010-03-17 21:52:07 +0100903 done = 1;
Willy Tarreauc1a07962010-03-16 20:55:43 +0100904 if (!s->check_data_len) {
905 if (!(s->result & SRV_CHK_ERROR))
906 set_server_check_status(s, HCHK_STATUS_SOCKERR, NULL);
907 goto out_wakeup;
908 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200909 }
910
Willy Tarreau03938182010-03-17 21:52:07 +0100911 /* Intermediate or complete response received.
912 * Terminate string in check_data buffer.
913 */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000914 if (s->check_data_len < BUFSIZE)
Willy Tarreau2c7ace02010-03-16 20:32:04 +0100915 s->check_data[s->check_data_len] = '\0';
Willy Tarreau03938182010-03-17 21:52:07 +0100916 else {
Willy Tarreau2c7ace02010-03-16 20:32:04 +0100917 s->check_data[s->check_data_len - 1] = '\0';
Willy Tarreau03938182010-03-17 21:52:07 +0100918 done = 1; /* buffer full, don't wait for more data */
919 }
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200920
Nick Chalk57b1bf72010-03-16 15:50:46 +0000921 /* Run the checks... */
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100922 if (s->proxy->options & PR_O_HTTP_CHK) {
Willy Tarreau03938182010-03-17 21:52:07 +0100923 if (!done && s->check_data_len < strlen("HTTP/1.0 000\r"))
924 goto wait_more_data;
925
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100926 /* Check if the server speaks HTTP 1.X */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000927 if ((s->check_data_len < strlen("HTTP/1.0 000\r")) ||
928 (memcmp(s->check_data, "HTTP/1.", 7) != 0 ||
929 (*(s->check_data + 12) != ' ' && *(s->check_data + 12) != '\r')) ||
930 !isdigit((unsigned char) *(s->check_data + 9)) || !isdigit((unsigned char) *(s->check_data + 10)) ||
931 !isdigit((unsigned char) *(s->check_data + 11))) {
932 cut_crlf(s->check_data);
933 set_server_check_status(s, HCHK_STATUS_L7RSP, s->check_data);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200934
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100935 goto out_wakeup;
936 }
937
Nick Chalk57b1bf72010-03-16 15:50:46 +0000938 s->check_code = str2uic(s->check_data + 9);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200939
Nick Chalk57b1bf72010-03-16 15:50:46 +0000940 desc = ltrim(s->check_data + 12, ' ');
941
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100942 /* check the reply : HTTP/1.X 2xx and 3xx are OK */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000943 if (*(s->check_data + 9) == '2' || *(s->check_data + 9) == '3') {
944 cut_crlf(desc);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200945 set_server_check_status(s, HCHK_STATUS_L7OKD, desc);
Nick Chalk57b1bf72010-03-16 15:50:46 +0000946 }
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200947 else if ((s->proxy->options & PR_O_DISABLE404) &&
Willy Tarreau48494c02007-11-30 10:41:39 +0100948 (s->state & SRV_RUNNING) &&
Nick Chalk57b1bf72010-03-16 15:50:46 +0000949 (s->check_code == 404)) {
950 /* 404 may be accepted as "stopping" only if the server was up */
951 cut_crlf(desc);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200952 set_server_check_status(s, HCHK_STATUS_L7OKCD, desc);
Nick Chalk57b1bf72010-03-16 15:50:46 +0000953 }
954 else {
955 cut_crlf(desc);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200956 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
Nick Chalk57b1bf72010-03-16 15:50:46 +0000957 }
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100958 }
Willy Tarreau07a54902010-03-29 18:33:29 +0200959 else if (s->proxy->options2 & PR_O2_SSL3_CHK) {
Willy Tarreau03938182010-03-17 21:52:07 +0100960 if (!done && s->check_data_len < 5)
961 goto wait_more_data;
962
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100963 /* Check for SSLv3 alert or handshake */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000964 if ((s->check_data_len >= 5) && (*s->check_data == 0x15 || *s->check_data == 0x16))
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200965 set_server_check_status(s, HCHK_STATUS_L6OK, NULL);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200966 else
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200967 set_server_check_status(s, HCHK_STATUS_L6RSP, NULL);
Willy Tarreau6996e152007-04-30 14:37:43 +0200968 }
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100969 else if (s->proxy->options & PR_O_SMTP_CHK) {
Willy Tarreau03938182010-03-17 21:52:07 +0100970 if (!done && s->check_data_len < strlen("000\r"))
971 goto wait_more_data;
972
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200973 /* Check if the server speaks SMTP */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000974 if ((s->check_data_len < strlen("000\r")) ||
975 (*(s->check_data + 3) != ' ' && *(s->check_data + 3) != '\r') ||
976 !isdigit((unsigned char) *s->check_data) || !isdigit((unsigned char) *(s->check_data + 1)) ||
977 !isdigit((unsigned char) *(s->check_data + 2))) {
978 cut_crlf(s->check_data);
979 set_server_check_status(s, HCHK_STATUS_L7RSP, s->check_data);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200980
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200981 goto out_wakeup;
982 }
983
Nick Chalk57b1bf72010-03-16 15:50:46 +0000984 s->check_code = str2uic(s->check_data);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200985
Nick Chalk57b1bf72010-03-16 15:50:46 +0000986 desc = ltrim(s->check_data + 3, ' ');
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200987 cut_crlf(desc);
988
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100989 /* Check for SMTP code 2xx (should be 250) */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000990 if (*s->check_data == '2')
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200991 set_server_check_status(s, HCHK_STATUS_L7OKD, desc);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200992 else
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200993 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
Willy Tarreau6996e152007-04-30 14:37:43 +0200994 }
Hervé COMMOWICK698ae002010-01-12 09:25:13 +0100995 else if (s->proxy->options2 & PR_O2_MYSQL_CHK) {
996 /* MySQL Error packet always begin with field_count = 0xff
997 * contrary to OK Packet who always begin whith 0x00 */
Willy Tarreau03938182010-03-17 21:52:07 +0100998 if (!done && s->check_data_len < 5)
999 goto wait_more_data;
1000
Nick Chalk57b1bf72010-03-16 15:50:46 +00001001 if (*(s->check_data + 4) != '\xff') {
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001002 /* We set the MySQL Version in description for information purpose
1003 * FIXME : it can be cool to use MySQL Version for other purpose,
1004 * like mark as down old MySQL server.
1005 */
Nick Chalk57b1bf72010-03-16 15:50:46 +00001006 if (s->check_data_len > 51) {
1007 desc = ltrim(s->check_data + 5, ' ');
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001008 set_server_check_status(s, HCHK_STATUS_L7OKD, desc);
1009 }
1010 else {
Willy Tarreau03938182010-03-17 21:52:07 +01001011 if (!done)
1012 goto wait_more_data;
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001013 /* it seems we have a OK packet but without a valid length,
1014 * it must be a protocol error
1015 */
Nick Chalk57b1bf72010-03-16 15:50:46 +00001016 set_server_check_status(s, HCHK_STATUS_L7RSP, s->check_data);
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001017 }
1018 }
1019 else {
1020 /* An error message is attached in the Error packet,
1021 * so we can display it ! :)
1022 */
Nick Chalk57b1bf72010-03-16 15:50:46 +00001023 desc = ltrim(s->check_data + 7, ' ');
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001024 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
1025 }
1026 }
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001027 else {
1028 /* other checks are valid if the connection succeeded anyway */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001029 set_server_check_status(s, HCHK_STATUS_L4OK, NULL);
Willy Tarreau23677902007-05-08 23:50:35 +02001030 }
Willy Tarreau83749182007-04-15 20:56:27 +02001031
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001032 out_wakeup:
1033 if (s->result & SRV_CHK_ERROR)
Willy Tarreaubaaee002006-06-26 02:48:02 +02001034 fdtab[fd].state = FD_STERROR;
1035
Nick Chalk57b1bf72010-03-16 15:50:46 +00001036 /* Reset the check buffer... */
1037 *s->check_data = '\0';
1038 s->check_data_len = 0;
1039
Willy Tarreau03938182010-03-17 21:52:07 +01001040 /* Close the connection... */
1041 shutdown(fd, SHUT_RDWR);
Willy Tarreauf161a342007-04-08 16:59:42 +02001042 EV_FD_CLR(fd, DIR_RD);
Willy Tarreaufdccded2008-08-29 18:19:04 +02001043 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreaud6f087e2008-01-18 17:20:13 +01001044 fdtab[fd].ev &= ~FD_POLL_IN;
Willy Tarreau83749182007-04-15 20:56:27 +02001045 return 1;
Willy Tarreau03938182010-03-17 21:52:07 +01001046
1047 wait_more_data:
1048 fdtab[fd].ev &= ~FD_POLL_IN;
1049 return 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001050}
1051
1052/*
1053 * manages a server health-check. Returns
1054 * the time the task accepts to wait, or TIME_ETERNITY for infinity.
1055 */
Willy Tarreau26c25062009-03-08 09:38:41 +01001056struct task *process_chk(struct task *t)
Willy Tarreaubaaee002006-06-26 02:48:02 +02001057{
Willy Tarreaue3838802009-03-21 18:58:32 +01001058 int attempts = 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001059 struct server *s = t->context;
1060 struct sockaddr_in sa;
1061 int fd;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001062 int rv;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001063
1064 //fprintf(stderr, "process_chk: task=%p\n", t);
1065
1066 new_chk:
Willy Tarreaue3838802009-03-21 18:58:32 +01001067 if (attempts++ > 0) {
1068 /* we always fail to create a server, let's stop insisting... */
1069 while (tick_is_expired(t->expire, now_ms))
1070 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
1071 return t;
1072 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001073 fd = s->curfd;
1074 if (fd < 0) { /* no check currently running */
1075 //fprintf(stderr, "process_chk: 2\n");
Willy Tarreau26c25062009-03-08 09:38:41 +01001076 if (!tick_is_expired(t->expire, now_ms)) /* woke up too early */
1077 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001078
1079 /* we don't send any health-checks when the proxy is stopped or when
1080 * the server should not be checked.
1081 */
Cyril Bontécd19e512010-01-31 22:34:03 +01001082 if (!(s->state & SRV_CHECKED) || s->proxy->state == PR_STSTOPPED || (s->state & SRV_MAINTAIN)) {
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001083 while (tick_is_expired(t->expire, now_ms))
1084 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
Willy Tarreau26c25062009-03-08 09:38:41 +01001085 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001086 }
1087
1088 /* we'll initiate a new check */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001089 set_server_check_status(s, HCHK_STATUS_START, NULL);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001090 if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) != -1) {
1091 if ((fd < global.maxsock) &&
1092 (fcntl(fd, F_SETFL, O_NONBLOCK) != -1) &&
1093 (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char *) &one, sizeof(one)) != -1)) {
1094 //fprintf(stderr, "process_chk: 3\n");
1095
Willy Tarreau9edd1612007-10-18 18:07:48 +02001096 if (s->proxy->options & PR_O_TCP_NOLING) {
1097 /* We don't want to useless data */
1098 setsockopt(fd, SOL_SOCKET, SO_LINGER, (struct linger *) &nolinger, sizeof(struct linger));
1099 }
Willy Tarreau2ea3abb2007-03-25 16:45:16 +02001100
Willy Tarreau0f03c6f2007-03-25 20:46:19 +02001101 if (s->check_addr.sin_addr.s_addr)
1102 /* we'll connect to the check addr specified on the server */
Willy Tarreau2ea3abb2007-03-25 16:45:16 +02001103 sa = s->check_addr;
Willy Tarreau2ea3abb2007-03-25 16:45:16 +02001104 else
Willy Tarreau0f03c6f2007-03-25 20:46:19 +02001105 /* we'll connect to the addr on the server */
Willy Tarreau2ea3abb2007-03-25 16:45:16 +02001106 sa = s->addr;
Willy Tarreau0f03c6f2007-03-25 20:46:19 +02001107
Willy Tarreaubaaee002006-06-26 02:48:02 +02001108 /* we'll connect to the check port on the server */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001109 sa.sin_port = htons(s->check_port);
1110
1111 /* allow specific binding :
1112 * - server-specific at first
1113 * - proxy-specific next
1114 */
1115 if (s->state & SRV_BIND_SRC) {
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001116 struct sockaddr_in *remote = NULL;
1117 int ret, flags = 0;
Willy Tarreau163c5322006-11-14 16:18:41 +01001118
Willy Tarreaucf1d5722008-02-14 20:28:18 +01001119#if defined(CONFIG_HAP_CTTPROXY) || defined(CONFIG_HAP_LINUX_TPROXY)
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001120 if ((s->state & SRV_TPROXY_MASK) == SRV_TPROXY_ADDR) {
1121 remote = (struct sockaddr_in *)&s->tproxy_addr;
1122 flags = 3;
1123 }
Willy Tarreaucf1d5722008-02-14 20:28:18 +01001124#endif
Willy Tarreauc76721d2009-02-04 20:20:58 +01001125#ifdef SO_BINDTODEVICE
1126 /* Note: this might fail if not CAP_NET_RAW */
1127 if (s->iface_name)
1128 setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE,
Willy Tarreau604e8302009-03-06 00:48:23 +01001129 s->iface_name, s->iface_len + 1);
Willy Tarreauc76721d2009-02-04 20:20:58 +01001130#endif
Willy Tarreauc6f4ce82009-06-10 11:09:37 +02001131 if (s->sport_range) {
1132 int bind_attempts = 10; /* should be more than enough to find a spare port */
1133 struct sockaddr_in src;
1134
1135 ret = 1;
1136 src = s->source_addr;
1137
1138 do {
1139 /* note: in case of retry, we may have to release a previously
1140 * allocated port, hence this loop's construct.
1141 */
Willy Tarreau8d5d77e2009-10-18 07:25:52 +02001142 port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
1143 fdinfo[fd].port_range = NULL;
Willy Tarreauc6f4ce82009-06-10 11:09:37 +02001144
1145 if (!bind_attempts)
1146 break;
1147 bind_attempts--;
1148
Willy Tarreau8d5d77e2009-10-18 07:25:52 +02001149 fdinfo[fd].local_port = port_range_alloc_port(s->sport_range);
1150 if (!fdinfo[fd].local_port)
Willy Tarreauc6f4ce82009-06-10 11:09:37 +02001151 break;
1152
Willy Tarreau8d5d77e2009-10-18 07:25:52 +02001153 fdinfo[fd].port_range = s->sport_range;
1154 src.sin_port = htons(fdinfo[fd].local_port);
Willy Tarreauc6f4ce82009-06-10 11:09:37 +02001155
1156 ret = tcpv4_bind_socket(fd, flags, &src, remote);
1157 } while (ret != 0); /* binding NOK */
1158 }
1159 else {
1160 ret = tcpv4_bind_socket(fd, flags, &s->source_addr, remote);
1161 }
1162
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001163 if (ret) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001164 set_server_check_status(s, HCHK_STATUS_SOCKERR, NULL);
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001165 switch (ret) {
1166 case 1:
1167 Alert("Cannot bind to source address before connect() for server %s/%s. Aborting.\n",
1168 s->proxy->id, s->id);
1169 break;
1170 case 2:
Willy Tarreau163c5322006-11-14 16:18:41 +01001171 Alert("Cannot bind to tproxy source address before connect() for server %s/%s. Aborting.\n",
1172 s->proxy->id, s->id);
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001173 break;
Willy Tarreau163c5322006-11-14 16:18:41 +01001174 }
1175 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001176 }
1177 else if (s->proxy->options & PR_O_BIND_SRC) {
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001178 struct sockaddr_in *remote = NULL;
1179 int ret, flags = 0;
1180
Willy Tarreaucf1d5722008-02-14 20:28:18 +01001181#if defined(CONFIG_HAP_CTTPROXY) || defined(CONFIG_HAP_LINUX_TPROXY)
Willy Tarreau163c5322006-11-14 16:18:41 +01001182 if ((s->proxy->options & PR_O_TPXY_MASK) == PR_O_TPXY_ADDR) {
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001183 remote = (struct sockaddr_in *)&s->proxy->tproxy_addr;
1184 flags = 3;
1185 }
Willy Tarreaucf1d5722008-02-14 20:28:18 +01001186#endif
Willy Tarreaud53f96b2009-02-04 18:46:54 +01001187#ifdef SO_BINDTODEVICE
1188 /* Note: this might fail if not CAP_NET_RAW */
1189 if (s->proxy->iface_name)
1190 setsockopt(fd, SOL_SOCKET, SO_BINDTODEVICE,
Willy Tarreau604e8302009-03-06 00:48:23 +01001191 s->proxy->iface_name, s->proxy->iface_len + 1);
Willy Tarreaud53f96b2009-02-04 18:46:54 +01001192#endif
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001193 ret = tcpv4_bind_socket(fd, flags, &s->proxy->source_addr, remote);
1194 if (ret) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001195 set_server_check_status(s, HCHK_STATUS_SOCKERR, NULL);
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001196 switch (ret) {
1197 case 1:
1198 Alert("Cannot bind to source address before connect() for %s '%s'. Aborting.\n",
1199 proxy_type_str(s->proxy), s->proxy->id);
1200 break;
1201 case 2:
Willy Tarreau2b5652f2006-12-31 17:46:05 +01001202 Alert("Cannot bind to tproxy source address before connect() for %s '%s'. Aborting.\n",
1203 proxy_type_str(s->proxy), s->proxy->id);
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001204 break;
Willy Tarreau163c5322006-11-14 16:18:41 +01001205 }
1206 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001207 }
1208
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001209 if (s->result == SRV_CHK_UNKNOWN) {
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +04001210#if defined(TCP_QUICKACK)
Willy Tarreau1274bc42009-07-15 07:16:31 +02001211 /* disabling tcp quick ack now allows
1212 * the request to leave the machine with
1213 * the first ACK.
1214 */
1215 if (s->proxy->options2 & PR_O2_SMARTCON)
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +04001216 setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, (char *) &zero, sizeof(zero));
Willy Tarreau1274bc42009-07-15 07:16:31 +02001217#endif
Willy Tarreaubaaee002006-06-26 02:48:02 +02001218 if ((connect(fd, (struct sockaddr *)&sa, sizeof(sa)) != -1) || (errno == EINPROGRESS)) {
1219 /* OK, connection in progress or established */
1220
1221 //fprintf(stderr, "process_chk: 4\n");
1222
1223 s->curfd = fd; /* that's how we know a test is in progress ;-) */
Willy Tarreau7a966482007-04-15 10:58:02 +02001224 fd_insert(fd);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001225 fdtab[fd].owner = t;
Willy Tarreau54469402006-07-29 16:59:06 +02001226 fdtab[fd].cb[DIR_RD].f = &event_srv_chk_r;
1227 fdtab[fd].cb[DIR_RD].b = NULL;
1228 fdtab[fd].cb[DIR_WR].f = &event_srv_chk_w;
1229 fdtab[fd].cb[DIR_WR].b = NULL;
Willy Tarreau8d5d77e2009-10-18 07:25:52 +02001230 fdinfo[fd].peeraddr = (struct sockaddr *)&sa;
1231 fdinfo[fd].peerlen = sizeof(sa);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001232 fdtab[fd].state = FD_STCONN; /* connection in progress */
Willy Tarreaufb14edc2009-06-14 15:24:37 +02001233 fdtab[fd].flags = FD_FL_TCP | FD_FL_TCP_NODELAY;
Willy Tarreauf161a342007-04-08 16:59:42 +02001234 EV_FD_SET(fd, DIR_WR); /* for connect status */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001235#ifdef DEBUG_FULL
Willy Tarreauf161a342007-04-08 16:59:42 +02001236 assert (!EV_FD_ISSET(fd, DIR_RD));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001237#endif
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001238 //fprintf(stderr, "process_chk: 4+, %lu\n", __tv_to_ms(&s->proxy->timeout.connect));
1239 /* we allow up to min(inter, timeout.connect) for a connection
1240 * to establish but only when timeout.check is set
1241 * as it may be to short for a full check otherwise
1242 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001243 t->expire = tick_add(now_ms, MS_TO_TICKS(s->inter));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001244
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001245 if (s->proxy->timeout.check && s->proxy->timeout.connect) {
1246 int t_con = tick_add(now_ms, s->proxy->timeout.connect);
1247 t->expire = tick_first(t->expire, t_con);
Willy Tarreau60548192008-02-17 11:34:10 +01001248 }
Willy Tarreau26c25062009-03-08 09:38:41 +01001249 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001250 }
1251 else if (errno != EALREADY && errno != EISCONN && errno != EAGAIN) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +02001252 /* a real error */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001253
1254 switch (errno) {
1255 /* FIXME: is it possible to get ECONNREFUSED/ENETUNREACH with O_NONBLOCK? */
1256 case ECONNREFUSED:
1257 case ENETUNREACH:
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001258 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(errno));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001259 break;
1260
1261 default:
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001262 set_server_check_status(s, HCHK_STATUS_SOCKERR, strerror(errno));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001263 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001264 }
1265 }
1266 }
Willy Tarreau8d5d77e2009-10-18 07:25:52 +02001267 port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
1268 fdinfo[fd].port_range = NULL;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001269 close(fd); /* socket creation error */
1270 }
1271
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001272 if (s->result == SRV_CHK_UNKNOWN) { /* nothing done */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001273 //fprintf(stderr, "process_chk: 6\n");
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001274 while (tick_is_expired(t->expire, now_ms))
1275 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001276 goto new_chk; /* may be we should initialize a new check */
1277 }
1278
1279 /* here, we have seen a failure */
1280 if (s->health > s->rise) {
1281 s->health--; /* still good */
Krzysztof Piotr Oledzki052d4fd2009-10-04 14:52:57 +02001282 s->counters.failed_checks++;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001283 }
1284 else
1285 set_server_down(s);
1286
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001287 //fprintf(stderr, "process_chk: 7, %lu\n", __tv_to_ms(&s->proxy->timeout.connect));
1288 /* we allow up to min(inter, timeout.connect) for a connection
1289 * to establish but only when timeout.check is set
1290 * as it may be to short for a full check otherwise
1291 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001292 while (tick_is_expired(t->expire, now_ms)) {
1293 int t_con;
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001294
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001295 t_con = tick_add(t->expire, s->proxy->timeout.connect);
1296 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001297
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001298 if (s->proxy->timeout.check)
1299 t->expire = tick_first(t->expire, t_con);
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001300 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001301 goto new_chk;
1302 }
1303 else {
1304 //fprintf(stderr, "process_chk: 8\n");
1305 /* there was a test running */
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001306 if ((s->result & (SRV_CHK_ERROR|SRV_CHK_RUNNING)) == SRV_CHK_RUNNING) { /* good server detected */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001307 //fprintf(stderr, "process_chk: 9\n");
Krzysztof Oledzki85130942007-10-22 16:21:10 +02001308
Willy Tarreau9909fc12007-11-30 17:42:05 +01001309 if (s->state & SRV_WARMINGUP) {
1310 if (now.tv_sec < s->last_change || now.tv_sec >= s->last_change + s->slowstart) {
1311 s->state &= ~SRV_WARMINGUP;
1312 if (s->proxy->lbprm.algo & BE_LB_PROP_DYN)
1313 s->eweight = s->uweight * BE_WEIGHT_SCALE;
1314 if (s->proxy->lbprm.update_server_eweight)
1315 s->proxy->lbprm.update_server_eweight(s);
1316 }
1317 else if (s->proxy->lbprm.algo & BE_LB_PROP_DYN) {
1318 /* for dynamic algorithms, let's update the weight */
Willy Tarreau5542af62007-12-03 02:04:00 +01001319 s->eweight = (BE_WEIGHT_SCALE * (now.tv_sec - s->last_change) +
1320 s->slowstart - 1) / s->slowstart;
Willy Tarreau9909fc12007-11-30 17:42:05 +01001321 s->eweight *= s->uweight;
1322 if (s->proxy->lbprm.update_server_eweight)
1323 s->proxy->lbprm.update_server_eweight(s);
1324 }
1325 /* probably that we can refill this server with a bit more connections */
1326 check_for_pending(s);
1327 }
1328
Willy Tarreau48494c02007-11-30 10:41:39 +01001329 /* we may have to add/remove this server from the LB group */
1330 if ((s->state & SRV_RUNNING) && (s->proxy->options & PR_O_DISABLE404)) {
1331 if ((s->state & SRV_GOINGDOWN) &&
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +01001332 ((s->result & (SRV_CHK_RUNNING|SRV_CHK_DISABLE)) == SRV_CHK_RUNNING))
1333 set_server_enabled(s);
Willy Tarreau48494c02007-11-30 10:41:39 +01001334 else if (!(s->state & SRV_GOINGDOWN) &&
1335 ((s->result & (SRV_CHK_RUNNING | SRV_CHK_DISABLE)) ==
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +01001336 (SRV_CHK_RUNNING | SRV_CHK_DISABLE)))
1337 set_server_disabled(s);
Willy Tarreau48494c02007-11-30 10:41:39 +01001338 }
1339
Krzysztof Oledzki85130942007-10-22 16:21:10 +02001340 if (s->health < s->rise + s->fall - 1) {
1341 s->health++; /* was bad, stays for a while */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001342
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +01001343 set_server_up(s);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001344 }
1345 s->curfd = -1; /* no check running anymore */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001346 fd_delete(fd);
Willy Tarreau44ec0f02007-10-14 23:47:04 +02001347
1348 rv = 0;
1349 if (global.spread_checks > 0) {
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001350 rv = srv_getinter(s) * global.spread_checks / 100;
Willy Tarreau44ec0f02007-10-14 23:47:04 +02001351 rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0)));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001352 //fprintf(stderr, "process_chk(%p): (%d+/-%d%%) random=%d\n", s, srv_getinter(s), global.spread_checks, rv);
Willy Tarreau44ec0f02007-10-14 23:47:04 +02001353 }
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001354 t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(s) + rv));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001355 goto new_chk;
1356 }
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001357 else if ((s->result & SRV_CHK_ERROR) || tick_is_expired(t->expire, now_ms)) {
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001358 if (!(s->result & SRV_CHK_ERROR)) {
1359 if (!EV_FD_ISSET(fd, DIR_RD)) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001360 set_server_check_status(s, HCHK_STATUS_L4TOUT, NULL);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001361 } else {
Willy Tarreau07a54902010-03-29 18:33:29 +02001362 if (s->proxy->options2 & PR_O2_SSL3_CHK)
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001363 set_server_check_status(s, HCHK_STATUS_L6TOUT, NULL);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001364 else /* HTTP, SMTP */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001365 set_server_check_status(s, HCHK_STATUS_L7TOUT, NULL);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001366 }
1367 }
1368
Willy Tarreaubaaee002006-06-26 02:48:02 +02001369 //fprintf(stderr, "process_chk: 10\n");
1370 /* failure or timeout detected */
1371 if (s->health > s->rise) {
1372 s->health--; /* still good */
Krzysztof Piotr Oledzki052d4fd2009-10-04 14:52:57 +02001373 s->counters.failed_checks++;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001374 }
1375 else
1376 set_server_down(s);
1377 s->curfd = -1;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001378 fd_delete(fd);
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001379
1380 rv = 0;
1381 if (global.spread_checks > 0) {
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001382 rv = srv_getinter(s) * global.spread_checks / 100;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001383 rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0)));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001384 //fprintf(stderr, "process_chk(%p): (%d+/-%d%%) random=%d\n", s, srv_getinter(s), global.spread_checks, rv);
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001385 }
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001386 t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(s) + rv));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001387 goto new_chk;
1388 }
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001389 /* if result is unknown and there's no timeout, we have to wait again */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001390 }
1391 //fprintf(stderr, "process_chk: 11\n");
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001392 s->result = SRV_CHK_UNKNOWN;
Willy Tarreau26c25062009-03-08 09:38:41 +01001393 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001394}
1395
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001396/*
1397 * Start health-check.
1398 * Returns 0 if OK, -1 if error, and prints the error in this case.
1399 */
1400int start_checks() {
1401
1402 struct proxy *px;
1403 struct server *s;
1404 struct task *t;
1405 int nbchk=0, mininter=0, srvpos=0;
1406
Willy Tarreau2c43a1e2007-10-14 23:05:39 +02001407 /* 1- count the checkers to run simultaneously.
1408 * We also determine the minimum interval among all of those which
1409 * have an interval larger than SRV_CHK_INTER_THRES. This interval
1410 * will be used to spread their start-up date. Those which have
1411 * a shorter interval will start independantly and will not dictate
1412 * too short an interval for all others.
1413 */
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001414 for (px = proxy; px; px = px->next) {
1415 for (s = px->srv; s; s = s->next) {
1416 if (!(s->state & SRV_CHECKED))
1417 continue;
1418
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001419 if ((srv_getinter(s) >= SRV_CHK_INTER_THRES) &&
1420 (!mininter || mininter > srv_getinter(s)))
1421 mininter = srv_getinter(s);
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001422
1423 nbchk++;
1424 }
1425 }
1426
1427 if (!nbchk)
1428 return 0;
1429
1430 srand((unsigned)time(NULL));
1431
1432 /*
1433 * 2- start them as far as possible from each others. For this, we will
1434 * start them after their interval set to the min interval divided by
1435 * the number of servers, weighted by the server's position in the list.
1436 */
1437 for (px = proxy; px; px = px->next) {
1438 for (s = px->srv; s; s = s->next) {
1439 if (!(s->state & SRV_CHECKED))
1440 continue;
1441
Willy Tarreaua4613182009-03-21 18:13:21 +01001442 if ((t = task_new()) == NULL) {
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001443 Alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
1444 return -1;
1445 }
1446
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +02001447 s->check = t;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001448 t->process = process_chk;
1449 t->context = s;
1450
1451 /* check this every ms */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001452 t->expire = tick_add(now_ms,
1453 MS_TO_TICKS(((mininter && mininter >= srv_getinter(s)) ?
1454 mininter : srv_getinter(s)) * srvpos / nbchk));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001455 s->check_start = now;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001456 task_queue(t);
1457
1458 srvpos++;
1459 }
1460 }
1461 return 0;
1462}
Willy Tarreaubaaee002006-06-26 02:48:02 +02001463
1464/*
1465 * Local variables:
1466 * c-indent-level: 8
1467 * c-basic-offset: 8
1468 * End:
1469 */