blob: f13273ed44ba7bb56a32b1ac2db5e07be0ae6475 [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
2 * Health-checks functions.
3 *
Willy Tarreau26c25062009-03-08 09:38:41 +01004 * Copyright 2000-2009 Willy Tarreau <w@1wt.eu>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02005 * Copyright 2007-2009 Krzysztof Piotr Oledzki <ole@ans.pl>
Willy Tarreaubaaee002006-06-26 02:48:02 +02006 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 *
12 */
13
Willy Tarreaub8816082008-01-18 12:18:15 +010014#include <assert.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020015#include <ctype.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020016#include <errno.h>
17#include <fcntl.h>
18#include <stdio.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020019#include <stdlib.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020020#include <string.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020021#include <time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020022#include <unistd.h>
23#include <sys/socket.h>
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +040024#include <sys/types.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020025#include <netinet/in.h>
Willy Tarreau1274bc42009-07-15 07:16:31 +020026#include <netinet/tcp.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020027#include <arpa/inet.h>
28
Willy Tarreauc7e42382012-08-24 19:22:53 +020029#include <common/chunk.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020030#include <common/compat.h>
31#include <common/config.h>
32#include <common/mini-clist.h>
Willy Tarreau83749182007-04-15 20:56:27 +020033#include <common/standard.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020034#include <common/time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020035
36#include <types/global.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020037
38#include <proto/backend.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020039#include <proto/checks.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020040#include <proto/fd.h>
41#include <proto/log.h>
42#include <proto/queue.h>
Willy Tarreauc6f4ce82009-06-10 11:09:37 +020043#include <proto/port_range.h>
Willy Tarreau3d300592007-03-18 18:34:41 +010044#include <proto/proto_http.h>
Willy Tarreaue8c66af2008-01-13 18:40:14 +010045#include <proto/proto_tcp.h>
Willy Tarreau2b5652f2006-12-31 17:46:05 +010046#include <proto/proxy.h>
Willy Tarreaufb56aab2012-09-28 14:40:02 +020047#include <proto/raw_sock.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020048#include <proto/server.h>
Simon Hormane0d1bfb2011-06-21 14:34:58 +090049#include <proto/session.h>
Willy Tarreau9e000c62011-03-10 14:03:36 +010050#include <proto/stream_interface.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020051#include <proto/task.h>
52
Willy Tarreaubd741542010-03-16 18:46:54 +010053static int httpchk_expect(struct server *s, int done);
54
Simon Horman63a4a822012-03-19 07:24:41 +090055static const struct check_status check_statuses[HCHK_STATUS_SIZE] = {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020056 [HCHK_STATUS_UNKNOWN] = { SRV_CHK_UNKNOWN, "UNK", "Unknown" },
57 [HCHK_STATUS_INI] = { SRV_CHK_UNKNOWN, "INI", "Initializing" },
58 [HCHK_STATUS_START] = { /* SPECIAL STATUS*/ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020059
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010060 [HCHK_STATUS_HANA] = { SRV_CHK_ERROR, "HANA", "Health analyze" },
61
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020062 [HCHK_STATUS_SOCKERR] = { SRV_CHK_ERROR, "SOCKERR", "Socket error" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020063
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020064 [HCHK_STATUS_L4OK] = { SRV_CHK_RUNNING, "L4OK", "Layer4 check passed" },
65 [HCHK_STATUS_L4TOUT] = { SRV_CHK_ERROR, "L4TOUT", "Layer4 timeout" },
66 [HCHK_STATUS_L4CON] = { SRV_CHK_ERROR, "L4CON", "Layer4 connection problem" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020067
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020068 [HCHK_STATUS_L6OK] = { SRV_CHK_RUNNING, "L6OK", "Layer6 check passed" },
69 [HCHK_STATUS_L6TOUT] = { SRV_CHK_ERROR, "L6TOUT", "Layer6 timeout" },
70 [HCHK_STATUS_L6RSP] = { SRV_CHK_ERROR, "L6RSP", "Layer6 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020071
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020072 [HCHK_STATUS_L7TOUT] = { SRV_CHK_ERROR, "L7TOUT", "Layer7 timeout" },
73 [HCHK_STATUS_L7RSP] = { SRV_CHK_ERROR, "L7RSP", "Layer7 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020074
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020075 [HCHK_STATUS_L57DATA] = { /* DUMMY STATUS */ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020076
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020077 [HCHK_STATUS_L7OKD] = { SRV_CHK_RUNNING, "L7OK", "Layer7 check passed" },
78 [HCHK_STATUS_L7OKCD] = { SRV_CHK_RUNNING | SRV_CHK_DISABLE, "L7OKC", "Layer7 check conditionally passed" },
79 [HCHK_STATUS_L7STS] = { SRV_CHK_ERROR, "L7STS", "Layer7 wrong status" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020080};
81
Simon Horman63a4a822012-03-19 07:24:41 +090082static const struct analyze_status analyze_statuses[HANA_STATUS_SIZE] = { /* 0: ignore, 1: error, 2: OK */
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010083 [HANA_STATUS_UNKNOWN] = { "Unknown", { 0, 0 }},
84
85 [HANA_STATUS_L4_OK] = { "L4 successful connection", { 2, 0 }},
86 [HANA_STATUS_L4_ERR] = { "L4 unsuccessful connection", { 1, 1 }},
87
88 [HANA_STATUS_HTTP_OK] = { "Correct http response", { 0, 2 }},
89 [HANA_STATUS_HTTP_STS] = { "Wrong http response", { 0, 1 }},
90 [HANA_STATUS_HTTP_HDRRSP] = { "Invalid http response (headers)", { 0, 1 }},
91 [HANA_STATUS_HTTP_RSP] = { "Invalid http response", { 0, 1 }},
92
93 [HANA_STATUS_HTTP_READ_ERROR] = { "Read error (http)", { 0, 1 }},
94 [HANA_STATUS_HTTP_READ_TIMEOUT] = { "Read timeout (http)", { 0, 1 }},
95 [HANA_STATUS_HTTP_BROKEN_PIPE] = { "Close from server (http)", { 0, 1 }},
96};
97
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020098/*
99 * Convert check_status code to description
100 */
101const char *get_check_status_description(short check_status) {
102
103 const char *desc;
104
105 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200106 desc = check_statuses[check_status].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200107 else
108 desc = NULL;
109
110 if (desc && *desc)
111 return desc;
112 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200113 return check_statuses[HCHK_STATUS_UNKNOWN].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200114}
115
116/*
117 * Convert check_status code to short info
118 */
119const char *get_check_status_info(short check_status) {
120
121 const char *info;
122
123 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200124 info = check_statuses[check_status].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200125 else
126 info = NULL;
127
128 if (info && *info)
129 return info;
130 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200131 return check_statuses[HCHK_STATUS_UNKNOWN].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200132}
133
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100134const char *get_analyze_status(short analyze_status) {
135
136 const char *desc;
137
138 if (analyze_status < HANA_STATUS_SIZE)
139 desc = analyze_statuses[analyze_status].desc;
140 else
141 desc = NULL;
142
143 if (desc && *desc)
144 return desc;
145 else
146 return analyze_statuses[HANA_STATUS_UNKNOWN].desc;
147}
148
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200149#define SSP_O_HCHK 0x0002
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200150
151static void server_status_printf(struct chunk *msg, struct server *s, unsigned options, int xferred) {
152
Willy Tarreau44267702011-10-28 15:35:33 +0200153 if (s->track)
Willy Tarreau77804732012-10-29 16:14:26 +0100154 chunk_appendf(msg, " via %s/%s",
Willy Tarreau44267702011-10-28 15:35:33 +0200155 s->track->proxy->id, s->track->id);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200156
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200157 if (options & SSP_O_HCHK) {
Willy Tarreau77804732012-10-29 16:14:26 +0100158 chunk_appendf(msg, ", reason: %s", get_check_status_description(s->check.status));
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200159
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200160 if (s->check.status >= HCHK_STATUS_L57DATA)
Willy Tarreau77804732012-10-29 16:14:26 +0100161 chunk_appendf(msg, ", code: %d", s->check.code);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200162
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200163 if (*s->check.desc) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200164 struct chunk src;
165
Willy Tarreau77804732012-10-29 16:14:26 +0100166 chunk_appendf(msg, ", info: \"");
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200167
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200168 chunk_initlen(&src, s->check.desc, 0, strlen(s->check.desc));
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200169 chunk_asciiencode(msg, &src, '"');
170
Willy Tarreau77804732012-10-29 16:14:26 +0100171 chunk_appendf(msg, "\"");
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200172 }
173
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200174 if (s->check.duration >= 0)
Willy Tarreau77804732012-10-29 16:14:26 +0100175 chunk_appendf(msg, ", check duration: %ldms", s->check.duration);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200176 }
177
Krzysztof Piotr Oledzki3bb05712010-09-27 13:10:50 +0200178 if (xferred >= 0) {
Krzysztof Piotr Oledzkib16a6072010-01-10 21:12:58 +0100179 if (!(s->state & SRV_RUNNING))
Willy Tarreau77804732012-10-29 16:14:26 +0100180 chunk_appendf(msg, ". %d active and %d backup servers left.%s"
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100181 " %d sessions active, %d requeued, %d remaining in queue",
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200182 s->proxy->srv_act, s->proxy->srv_bck,
183 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
184 s->cur_sess, xferred, s->nbpend);
185 else
Willy Tarreau77804732012-10-29 16:14:26 +0100186 chunk_appendf(msg, ". %d active and %d backup servers online.%s"
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100187 " %d sessions requeued, %d total in queue",
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200188 s->proxy->srv_act, s->proxy->srv_bck,
189 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
190 xferred, s->nbpend);
191 }
192}
193
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200194/*
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200195 * Set s->check.status, update s->check.duration and fill s->result with
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200196 * an adequate SRV_CHK_* value.
197 *
198 * Show information in logs about failed health check if server is UP
199 * or succeeded health checks if server is DOWN.
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200200 */
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100201static void set_server_check_status(struct server *s, short status, char *desc)
202{
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200203 if (status == HCHK_STATUS_START) {
204 s->result = SRV_CHK_UNKNOWN; /* no result yet */
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200205 s->check.desc[0] = '\0';
206 s->check.start = now;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200207 return;
208 }
209
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200210 if (!s->check.status)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200211 return;
212
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200213 if (desc && *desc) {
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200214 strncpy(s->check.desc, desc, HCHK_DESC_LEN-1);
215 s->check.desc[HCHK_DESC_LEN-1] = '\0';
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200216 } else
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200217 s->check.desc[0] = '\0';
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200218
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200219 s->check.status = status;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200220 if (check_statuses[status].result)
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100221 s->result = check_statuses[status].result;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200222
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100223 if (status == HCHK_STATUS_HANA)
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200224 s->check.duration = -1;
225 else if (!tv_iszero(&s->check.start)) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200226 /* set_server_check_status() may be called more than once */
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200227 s->check.duration = tv_ms_elapsed(&s->check.start, &now);
228 tv_zero(&s->check.start);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200229 }
230
231 if (s->proxy->options2 & PR_O2_LOGHCHKS &&
232 (((s->health != 0) && (s->result & SRV_CHK_ERROR)) ||
233 ((s->health != s->rise + s->fall - 1) && (s->result & SRV_CHK_RUNNING)) ||
234 ((s->state & SRV_GOINGDOWN) && !(s->result & SRV_CHK_DISABLE)) ||
235 (!(s->state & SRV_GOINGDOWN) && (s->result & SRV_CHK_DISABLE)))) {
236
237 int health, rise, fall, state;
238
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100239 chunk_reset(&trash);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200240
241 /* FIXME begin: calculate local version of the health/rise/fall/state */
242 health = s->health;
243 rise = s->rise;
244 fall = s->fall;
245 state = s->state;
246
247 if (s->result & SRV_CHK_ERROR) {
248 if (health > rise) {
249 health--; /* still good */
250 } else {
251 if (health == rise)
252 state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
253
254 health = 0;
255 }
256 }
257
258 if (s->result & SRV_CHK_RUNNING) {
259 if (health < rise + fall - 1) {
260 health++; /* was bad, stays for a while */
261
262 if (health == rise)
263 state |= SRV_RUNNING;
264
265 if (health >= rise)
266 health = rise + fall - 1; /* OK now */
267 }
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100268
269 /* clear consecutive_errors if observing is enabled */
270 if (s->onerror)
271 s->consecutive_errors = 0;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200272 }
273 /* FIXME end: calculate local version of the health/rise/fall/state */
274
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100275 chunk_appendf(&trash,
276 "Health check for %sserver %s/%s %s%s",
277 s->state & SRV_BACKUP ? "backup " : "",
278 s->proxy->id, s->id,
279 (s->result & SRV_CHK_DISABLE)?"conditionally ":"",
280 (s->result & SRV_CHK_RUNNING)?"succeeded":"failed");
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200281
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100282 server_status_printf(&trash, s, SSP_O_HCHK, -1);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200283
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100284 chunk_appendf(&trash, ", status: %d/%d %s",
285 (state & SRV_RUNNING) ? (health - rise + 1) : (health),
286 (state & SRV_RUNNING) ? (fall) : (rise),
287 (state & SRV_RUNNING)?"UP":"DOWN");
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200288
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100289 Warning("%s.\n", trash.str);
290 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200291 }
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200292}
293
Willy Tarreau48494c02007-11-30 10:41:39 +0100294/* sends a log message when a backend goes down, and also sets last
295 * change date.
296 */
297static void set_backend_down(struct proxy *be)
298{
299 be->last_change = now.tv_sec;
300 be->down_trans++;
301
302 Alert("%s '%s' has no server available!\n", proxy_type_str(be), be->id);
303 send_log(be, LOG_EMERG, "%s %s has no server available!\n", proxy_type_str(be), be->id);
304}
305
306/* Redistribute pending connections when a server goes down. The number of
307 * connections redistributed is returned.
308 */
309static int redistribute_pending(struct server *s)
310{
311 struct pendconn *pc, *pc_bck, *pc_end;
312 int xferred = 0;
313
314 FOREACH_ITEM_SAFE(pc, pc_bck, &s->pendconns, pc_end, struct pendconn *, list) {
315 struct session *sess = pc->sess;
Willy Tarreau4de91492010-01-22 19:10:05 +0100316 if ((sess->be->options & (PR_O_REDISP|PR_O_PERSIST)) == PR_O_REDISP &&
317 !(sess->flags & SN_FORCE_PRST)) {
Willy Tarreau48494c02007-11-30 10:41:39 +0100318 /* The REDISP option was specified. We will ignore
319 * cookie and force to balance or use the dispatcher.
320 */
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100321
Krzysztof Piotr Oledzki5a329cf2008-02-22 03:50:19 +0100322 /* it's left to the dispatcher to choose a server */
Willy Tarreau48494c02007-11-30 10:41:39 +0100323 sess->flags &= ~(SN_DIRECT | SN_ASSIGNED | SN_ADDR_SET);
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100324
Willy Tarreau48494c02007-11-30 10:41:39 +0100325 pendconn_free(pc);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200326 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100327 xferred++;
328 }
329 }
330 return xferred;
331}
332
333/* Check for pending connections at the backend, and assign some of them to
334 * the server coming up. The server's weight is checked before being assigned
335 * connections it may not be able to handle. The total number of transferred
336 * connections is returned.
337 */
338static int check_for_pending(struct server *s)
339{
340 int xferred;
341
342 if (!s->eweight)
343 return 0;
344
345 for (xferred = 0; !s->maxconn || xferred < srv_dynamic_maxconn(s); xferred++) {
346 struct session *sess;
347 struct pendconn *p;
348
349 p = pendconn_from_px(s->proxy);
350 if (!p)
351 break;
Willy Tarreau3fdb3662012-11-12 00:42:33 +0100352 p->sess->target = &s->obj_type;
Willy Tarreau48494c02007-11-30 10:41:39 +0100353 sess = p->sess;
354 pendconn_free(p);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200355 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100356 }
357 return xferred;
358}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200359
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700360/* Shutdown all connections of a server. The caller must pass a termination
361 * code in <why>, which must be one of SN_ERR_* indicating the reason for the
362 * shutdown.
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900363 */
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700364static void shutdown_sessions(struct server *srv, int why)
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900365{
366 struct session *session, *session_bck;
367
Willy Tarreaua2a64e92011-09-07 23:01:56 +0200368 list_for_each_entry_safe(session, session_bck, &srv->actconns, by_srv)
369 if (session->srv_conn == srv)
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700370 session_shutdown(session, why);
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900371}
372
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700373/* Shutdown all connections of all backup servers of a proxy. The caller must
374 * pass a termination code in <why>, which must be one of SN_ERR_* indicating
375 * the reason for the shutdown.
376 */
377static void shutdown_backup_sessions(struct proxy *px, int why)
378{
379 struct server *srv;
380
381 for (srv = px->srv; srv != NULL; srv = srv->next)
382 if (srv->state & SRV_BACKUP)
383 shutdown_sessions(srv, why);
384}
385
Willy Tarreaubaaee002006-06-26 02:48:02 +0200386/* Sets server <s> down, notifies by all available means, recounts the
387 * remaining servers on the proxy and transfers queued sessions whenever
Willy Tarreau5af3a692007-07-24 23:32:33 +0200388 * possible to other servers. It automatically recomputes the number of
389 * servers, but not the map.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200390 */
Cyril Bontécd19e512010-01-31 22:34:03 +0100391void set_server_down(struct server *s)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200392{
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100393 struct server *srv;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200394 int xferred;
395
Cyril Bontécd19e512010-01-31 22:34:03 +0100396 if (s->state & SRV_MAINTAIN) {
397 s->health = s->rise;
398 }
399
Willy Tarreau44267702011-10-28 15:35:33 +0200400 if (s->health == s->rise || s->track) {
Willy Tarreau48494c02007-11-30 10:41:39 +0100401 int srv_was_paused = s->state & SRV_GOINGDOWN;
Willy Tarreaud64d2252010-10-17 17:16:42 +0200402 int prev_srv_count = s->proxy->srv_bck + s->proxy->srv_act;
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200403
404 s->last_change = now.tv_sec;
Willy Tarreau48494c02007-11-30 10:41:39 +0100405 s->state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
Willy Tarreau9580d162012-05-19 19:07:40 +0200406 if (s->proxy->lbprm.set_server_status_down)
407 s->proxy->lbprm.set_server_status_down(s);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200408
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900409 if (s->onmarkeddown & HANA_ONMARKEDDOWN_SHUTDOWNSESSIONS)
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700410 shutdown_sessions(s, SN_ERR_DOWN);
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900411
Willy Tarreaubaaee002006-06-26 02:48:02 +0200412 /* we might have sessions queued on this server and waiting for
413 * a connection. Those which are redispatchable will be queued
414 * to another server or to the proxy itself.
415 */
Willy Tarreau48494c02007-11-30 10:41:39 +0100416 xferred = redistribute_pending(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100417
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100418 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100419
Cyril Bontécd19e512010-01-31 22:34:03 +0100420 if (s->state & SRV_MAINTAIN) {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100421 chunk_appendf(&trash,
422 "%sServer %s/%s is DOWN for maintenance", s->state & SRV_BACKUP ? "Backup " : "",
423 s->proxy->id, s->id);
Cyril Bontécd19e512010-01-31 22:34:03 +0100424 } else {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100425 chunk_appendf(&trash,
426 "%sServer %s/%s is DOWN", s->state & SRV_BACKUP ? "Backup " : "",
427 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100428
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100429 server_status_printf(&trash, s,
430 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? SSP_O_HCHK : 0),
431 xferred);
Cyril Bontécd19e512010-01-31 22:34:03 +0100432 }
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100433 Warning("%s.\n", trash.str);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200434
Willy Tarreau48494c02007-11-30 10:41:39 +0100435 /* we don't send an alert if the server was previously paused */
436 if (srv_was_paused)
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100437 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Willy Tarreau48494c02007-11-30 10:41:39 +0100438 else
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100439 send_log(s->proxy, LOG_ALERT, "%s.\n", trash.str);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200440
Willy Tarreaud64d2252010-10-17 17:16:42 +0200441 if (prev_srv_count && s->proxy->srv_bck == 0 && s->proxy->srv_act == 0)
Willy Tarreau48494c02007-11-30 10:41:39 +0100442 set_backend_down(s->proxy);
443
Krzysztof Piotr Oledzki052d4fd2009-10-04 14:52:57 +0200444 s->counters.down_trans++;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100445
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100446 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100447 for(srv = s->tracknext; srv; srv = srv->tracknext)
Cyril Bontécd19e512010-01-31 22:34:03 +0100448 if (! (srv->state & SRV_MAINTAIN))
449 /* Only notify tracking servers that are not already in maintenance. */
450 set_server_down(srv);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200451 }
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100452
Willy Tarreaubaaee002006-06-26 02:48:02 +0200453 s->health = 0; /* failure */
454}
455
Cyril Bontécd19e512010-01-31 22:34:03 +0100456void set_server_up(struct server *s) {
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100457
458 struct server *srv;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100459 int xferred;
Willy Tarreau45446782012-03-09 17:16:09 +0100460 unsigned int old_state = s->state;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100461
Cyril Bontécd19e512010-01-31 22:34:03 +0100462 if (s->state & SRV_MAINTAIN) {
463 s->health = s->rise;
464 }
465
Willy Tarreau44267702011-10-28 15:35:33 +0200466 if (s->health == s->rise || s->track) {
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100467 if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0) {
468 if (s->proxy->last_change < now.tv_sec) // ignore negative times
469 s->proxy->down_time += now.tv_sec - s->proxy->last_change;
470 s->proxy->last_change = now.tv_sec;
471 }
472
473 if (s->last_change < now.tv_sec) // ignore negative times
474 s->down_time += now.tv_sec - s->last_change;
475
476 s->last_change = now.tv_sec;
477 s->state |= SRV_RUNNING;
Willy Tarreau45446782012-03-09 17:16:09 +0100478 s->state &= ~SRV_MAINTAIN;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100479
480 if (s->slowstart > 0) {
481 s->state |= SRV_WARMINGUP;
482 if (s->proxy->lbprm.algo & BE_LB_PROP_DYN) {
483 /* For dynamic algorithms, start at the first step of the weight,
484 * without multiplying by BE_WEIGHT_SCALE.
485 */
486 s->eweight = s->uweight;
487 if (s->proxy->lbprm.update_server_eweight)
488 s->proxy->lbprm.update_server_eweight(s);
489 }
Willy Tarreau2e993902011-10-31 11:53:20 +0100490 task_schedule(s->warmup, tick_add(now_ms, MS_TO_TICKS(MAX(1000, s->slowstart / 20))));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100491 }
Willy Tarreau9580d162012-05-19 19:07:40 +0200492 if (s->proxy->lbprm.set_server_status_up)
493 s->proxy->lbprm.set_server_status_up(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100494
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700495 /* If the server is set with "on-marked-up shutdown-backup-sessions",
496 * and it's not a backup server and its effective weight is > 0,
497 * then it can accept new connections, so we shut down all sessions
498 * on all backup servers.
499 */
500 if ((s->onmarkedup & HANA_ONMARKEDUP_SHUTDOWNBACKUPSESSIONS) &&
501 !(s->state & SRV_BACKUP) && s->eweight)
502 shutdown_backup_sessions(s->proxy, SN_ERR_UP);
503
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100504 /* check if we can handle some connections queued at the proxy. We
505 * will take as many as we can handle.
506 */
507 xferred = check_for_pending(s);
508
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100509 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100510
Willy Tarreau45446782012-03-09 17:16:09 +0100511 if (old_state & SRV_MAINTAIN) {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100512 chunk_appendf(&trash,
513 "%sServer %s/%s is UP (leaving maintenance)", s->state & SRV_BACKUP ? "Backup " : "",
514 s->proxy->id, s->id);
Cyril Bontécd19e512010-01-31 22:34:03 +0100515 } else {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100516 chunk_appendf(&trash,
517 "%sServer %s/%s is UP", s->state & SRV_BACKUP ? "Backup " : "",
518 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100519
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100520 server_status_printf(&trash, s,
521 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? SSP_O_HCHK : 0),
522 xferred);
Cyril Bontécd19e512010-01-31 22:34:03 +0100523 }
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100524
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100525 Warning("%s.\n", trash.str);
526 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100527
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100528 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100529 for(srv = s->tracknext; srv; srv = srv->tracknext)
Cyril Bontécd19e512010-01-31 22:34:03 +0100530 if (! (srv->state & SRV_MAINTAIN))
531 /* Only notify tracking servers if they're not in maintenance. */
532 set_server_up(srv);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100533 }
534
535 if (s->health >= s->rise)
536 s->health = s->rise + s->fall - 1; /* OK now */
537
538}
539
540static void set_server_disabled(struct server *s) {
541
542 struct server *srv;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100543 int xferred;
544
545 s->state |= SRV_GOINGDOWN;
Willy Tarreau9580d162012-05-19 19:07:40 +0200546 if (s->proxy->lbprm.set_server_status_down)
547 s->proxy->lbprm.set_server_status_down(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100548
549 /* we might have sessions queued on this server and waiting for
550 * a connection. Those which are redispatchable will be queued
551 * to another server or to the proxy itself.
552 */
553 xferred = redistribute_pending(s);
554
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100555 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100556
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100557 chunk_appendf(&trash,
558 "Load-balancing on %sServer %s/%s is disabled",
559 s->state & SRV_BACKUP ? "Backup " : "",
560 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100561
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100562 server_status_printf(&trash, s,
563 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? SSP_O_HCHK : 0),
564 xferred);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100565
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100566 Warning("%s.\n", trash.str);
567 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100568
569 if (!s->proxy->srv_bck && !s->proxy->srv_act)
570 set_backend_down(s->proxy);
571
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100572 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100573 for(srv = s->tracknext; srv; srv = srv->tracknext)
574 set_server_disabled(srv);
575}
576
577static void set_server_enabled(struct server *s) {
578
579 struct server *srv;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100580 int xferred;
581
582 s->state &= ~SRV_GOINGDOWN;
Willy Tarreau9580d162012-05-19 19:07:40 +0200583 if (s->proxy->lbprm.set_server_status_up)
584 s->proxy->lbprm.set_server_status_up(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100585
586 /* check if we can handle some connections queued at the proxy. We
587 * will take as many as we can handle.
588 */
589 xferred = check_for_pending(s);
590
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100591 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100592
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100593 chunk_appendf(&trash,
594 "Load-balancing on %sServer %s/%s is enabled again",
595 s->state & SRV_BACKUP ? "Backup " : "",
596 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100597
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100598 server_status_printf(&trash, s,
599 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? SSP_O_HCHK : 0),
600 xferred);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100601
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100602 Warning("%s.\n", trash.str);
603 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100604
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100605 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100606 for(srv = s->tracknext; srv; srv = srv->tracknext)
607 set_server_enabled(srv);
608}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200609
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100610void health_adjust(struct server *s, short status)
611{
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100612 int failed;
613 int expire;
614
615 /* return now if observing nor health check is not enabled */
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200616 if (!s->observe || !s->check.task)
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100617 return;
618
619 if (s->observe >= HANA_OBS_SIZE)
620 return;
621
622 if (status >= HCHK_STATUS_SIZE || !analyze_statuses[status].desc)
623 return;
624
625 switch (analyze_statuses[status].lr[s->observe - 1]) {
626 case 1:
627 failed = 1;
628 break;
629
630 case 2:
631 failed = 0;
632 break;
633
634 default:
635 return;
636 }
637
638 if (!failed) {
639 /* good: clear consecutive_errors */
640 s->consecutive_errors = 0;
641 return;
642 }
643
644 s->consecutive_errors++;
645
646 if (s->consecutive_errors < s->consecutive_errors_limit)
647 return;
648
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100649 chunk_printf(&trash, "Detected %d consecutive errors, last one was: %s",
650 s->consecutive_errors, get_analyze_status(status));
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100651
652 switch (s->onerror) {
653 case HANA_ONERR_FASTINTER:
654 /* force fastinter - nothing to do here as all modes force it */
655 break;
656
657 case HANA_ONERR_SUDDTH:
658 /* simulate a pre-fatal failed health check */
659 if (s->health > s->rise)
660 s->health = s->rise + 1;
661
662 /* no break - fall through */
663
664 case HANA_ONERR_FAILCHK:
665 /* simulate a failed health check */
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100666 set_server_check_status(s, HCHK_STATUS_HANA, trash.str);
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100667
668 if (s->health > s->rise) {
669 s->health--; /* still good */
670 s->counters.failed_checks++;
671 }
672 else
673 set_server_down(s);
674
675 break;
676
677 case HANA_ONERR_MARKDWN:
678 /* mark server down */
679 s->health = s->rise;
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100680 set_server_check_status(s, HCHK_STATUS_HANA, trash.str);
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100681 set_server_down(s);
682
683 break;
684
685 default:
686 /* write a warning? */
687 break;
688 }
689
690 s->consecutive_errors = 0;
691 s->counters.failed_hana++;
692
693 if (s->fastinter) {
694 expire = tick_add(now_ms, MS_TO_TICKS(s->fastinter));
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200695 if (s->check.task->expire > expire)
696 s->check.task->expire = expire;
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100697 }
698}
699
Willy Tarreauef781042010-01-27 11:53:01 +0100700static int httpchk_build_status_header(struct server *s, char *buffer)
701{
702 int sv_state;
703 int ratio;
704 int hlen = 0;
705 const char *srv_hlt_st[7] = { "DOWN", "DOWN %d/%d",
706 "UP %d/%d", "UP",
707 "NOLB %d/%d", "NOLB",
708 "no check" };
709
710 memcpy(buffer + hlen, "X-Haproxy-Server-State: ", 24);
711 hlen += 24;
712
713 if (!(s->state & SRV_CHECKED))
714 sv_state = 6; /* should obviously never happen */
715 else if (s->state & SRV_RUNNING) {
716 if (s->health == s->rise + s->fall - 1)
717 sv_state = 3; /* UP */
718 else
719 sv_state = 2; /* going down */
720
721 if (s->state & SRV_GOINGDOWN)
722 sv_state += 2;
723 } else {
724 if (s->health)
725 sv_state = 1; /* going up */
726 else
727 sv_state = 0; /* DOWN */
728 }
729
730 hlen += sprintf(buffer + hlen,
731 srv_hlt_st[sv_state],
732 (s->state & SRV_RUNNING) ? (s->health - s->rise + 1) : (s->health),
733 (s->state & SRV_RUNNING) ? (s->fall) : (s->rise));
734
735 hlen += sprintf(buffer + hlen, "; name=%s/%s; node=%s; weight=%d/%d; scur=%d/%d; qcur=%d",
736 s->proxy->id, s->id,
737 global.node,
738 (s->eweight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv,
739 (s->proxy->lbprm.tot_weight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv,
740 s->cur_sess, s->proxy->beconn - s->proxy->nbpend,
741 s->nbpend);
742
743 if ((s->state & SRV_WARMINGUP) &&
744 now.tv_sec < s->last_change + s->slowstart &&
745 now.tv_sec >= s->last_change) {
746 ratio = MAX(1, 100 * (now.tv_sec - s->last_change) / s->slowstart);
747 hlen += sprintf(buffer + hlen, "; throttle=%d%%", ratio);
748 }
749
750 buffer[hlen++] = '\r';
751 buffer[hlen++] = '\n';
752
753 return hlen;
754}
755
Willy Tarreaubaaee002006-06-26 02:48:02 +0200756/*
757 * This function is used only for server health-checks. It handles
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200758 * the connection acknowledgement. If the proxy requires L7 health-checks,
759 * it sends the request. In other cases, it calls set_server_check_status()
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200760 * to set s->check.status, s->check.duration and s->result.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200761 */
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200762static void event_srv_chk_w(struct connection *conn)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200763{
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200764 struct server *s = conn->owner;
765 int fd = conn->t.sock.fd;
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200766 struct task *t = s->check.task;
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200767
Willy Tarreauf1503172012-09-28 19:39:36 +0200768 if (conn->flags & (CO_FL_SOCK_WR_SH | CO_FL_DATA_WR_SH | CO_FL_WAIT_DATA | CO_FL_WAIT_WR))
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200769 conn->flags |= CO_FL_ERROR;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200770
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200771 if (unlikely(conn->flags & CO_FL_ERROR)) {
Krzysztof Piotr Oledzki6492db52010-01-02 22:03:01 +0100772 int skerr, err = errno;
773 socklen_t lskerr = sizeof(skerr);
774
775 if (!getsockopt(fd, SOL_SOCKET, SO_ERROR, &skerr, &lskerr) && skerr)
776 err = skerr;
777
778 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(err));
Willy Tarreau6996e152007-04-30 14:37:43 +0200779 goto out_error;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200780 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200781
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200782 if (conn->flags & CO_FL_HANDSHAKE)
783 return;
Willy Tarreau83749182007-04-15 20:56:27 +0200784
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200785 /* here, we know that the connection is established */
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100786 if (!(s->result & SRV_CHK_ERROR)) {
Willy Tarreaubaaee002006-06-26 02:48:02 +0200787 /* we don't want to mark 'UP' a server on which we detected an error earlier */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200788 if (s->check.bo->o) {
Willy Tarreau392e9392012-10-05 14:54:30 +0200789 conn->xprt->snd_buf(conn, s->check.bo, MSG_DONTWAIT | MSG_NOSIGNAL);
Willy Tarreauf1503172012-09-28 19:39:36 +0200790 if (conn->flags & CO_FL_ERROR) {
791 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(errno));
792 goto out_wakeup;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200793 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200794 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200795
Willy Tarreauf1503172012-09-28 19:39:36 +0200796 if (!s->check.bo->o) {
797 /* full request sent, we allow up to <timeout.check> if nonzero for a response */
798 if (s->proxy->timeout.check) {
799 t->expire = tick_add_ifset(now_ms, s->proxy->timeout.check);
800 task_queue(t);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200801 }
Willy Tarreauf1503172012-09-28 19:39:36 +0200802 __conn_data_want_recv(conn); /* prepare for reading reply */
803 goto out_nowake;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200804 }
Willy Tarreauf1503172012-09-28 19:39:36 +0200805 goto out_poll;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200806 }
Willy Tarreau83749182007-04-15 20:56:27 +0200807 out_wakeup:
Willy Tarreaufdccded2008-08-29 18:19:04 +0200808 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreau83749182007-04-15 20:56:27 +0200809 out_nowake:
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200810 __conn_data_stop_send(conn); /* nothing more to write */
Willy Tarreau3267d362012-08-17 23:53:56 +0200811 return;
Willy Tarreau6996e152007-04-30 14:37:43 +0200812 out_poll:
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200813 __conn_data_poll_send(conn);
Willy Tarreau3267d362012-08-17 23:53:56 +0200814 return;
Willy Tarreau6996e152007-04-30 14:37:43 +0200815 out_error:
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200816 conn->flags |= CO_FL_ERROR;
Willy Tarreau6996e152007-04-30 14:37:43 +0200817 goto out_wakeup;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200818}
819
820
821/*
Willy Tarreauf3c69202006-07-09 16:42:34 +0200822 * This function is used only for server health-checks. It handles the server's
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +0200823 * reply to an HTTP request, SSL HELLO or MySQL client Auth. It calls
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200824 * set_server_check_status() to update s->check.status, s->check.duration
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +0200825 * and s->result.
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200826
827 * The set_server_check_status function is called with HCHK_STATUS_L7OKD if
828 * an HTTP server replies HTTP 2xx or 3xx (valid responses), if an SMTP server
829 * returns 2xx, HCHK_STATUS_L6OK if an SSL server returns at least 5 bytes in
830 * response to an SSL HELLO (the principle is that this is enough to
831 * distinguish between an SSL server and a pure TCP relay). All other cases will
832 * call it with a proper error status like HCHK_STATUS_L7STS, HCHK_STATUS_L6RSP,
833 * etc.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200834 */
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200835static void event_srv_chk_r(struct connection *conn)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200836{
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200837 struct server *s = conn->owner;
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200838 struct task *t = s->check.task;
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200839 char *desc;
Willy Tarreau03938182010-03-17 21:52:07 +0100840 int done;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +0200841 unsigned short msglen;
Willy Tarreau83749182007-04-15 20:56:27 +0200842
Willy Tarreauf1503172012-09-28 19:39:36 +0200843 if (conn->flags & (CO_FL_SOCK_WR_SH | CO_FL_DATA_WR_SH | CO_FL_WAIT_DATA | CO_FL_WAIT_WR))
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200844 conn->flags |= CO_FL_ERROR;
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200845
846 if (unlikely((s->result & SRV_CHK_ERROR) || (conn->flags & CO_FL_ERROR))) {
Willy Tarreau83749182007-04-15 20:56:27 +0200847 /* in case of TCP only, this tells us if the connection failed */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200848 if (!(s->result & SRV_CHK_ERROR))
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200849 set_server_check_status(s, HCHK_STATUS_SOCKERR, NULL);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200850
Willy Tarreau83749182007-04-15 20:56:27 +0200851 goto out_wakeup;
852 }
853
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200854 if (conn->flags & CO_FL_HANDSHAKE)
855 return;
856
Willy Tarreau83749182007-04-15 20:56:27 +0200857 /* Warning! Linux returns EAGAIN on SO_ERROR if data are still available
858 * but the connection was closed on the remote end. Fortunately, recv still
859 * works correctly and we don't need to do the getsockopt() on linux.
860 */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000861
862 /* Set buffer to point to the end of the data already read, and check
863 * that there is free space remaining. If the buffer is full, proceed
864 * with running the checks without attempting another socket read.
865 */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000866
Willy Tarreau03938182010-03-17 21:52:07 +0100867 done = 0;
Nick Chalk57b1bf72010-03-16 15:50:46 +0000868
Willy Tarreau392e9392012-10-05 14:54:30 +0200869 conn->xprt->rcv_buf(conn, s->check.bi, s->check.bi->size);
Willy Tarreauf1503172012-09-28 19:39:36 +0200870 if (conn->flags & (CO_FL_ERROR | CO_FL_SOCK_RD_SH | CO_FL_DATA_RD_SH)) {
Willy Tarreau03938182010-03-17 21:52:07 +0100871 done = 1;
Willy Tarreauf1503172012-09-28 19:39:36 +0200872 if ((conn->flags & CO_FL_ERROR) && !s->check.bi->i) {
873 /* Report network errors only if we got no other data. Otherwise
874 * we'll let the upper layers decide whether the response is OK
875 * or not. It is very common that an RST sent by the server is
876 * reported as an error just after the last data chunk.
877 */
Willy Tarreauc1a07962010-03-16 20:55:43 +0100878 if (!(s->result & SRV_CHK_ERROR))
879 set_server_check_status(s, HCHK_STATUS_SOCKERR, NULL);
880 goto out_wakeup;
881 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200882 }
883
Willy Tarreau03938182010-03-17 21:52:07 +0100884 /* Intermediate or complete response received.
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200885 * Terminate string in check.bi->data buffer.
Willy Tarreau03938182010-03-17 21:52:07 +0100886 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200887 if (s->check.bi->i < s->check.bi->size)
888 s->check.bi->data[s->check.bi->i] = '\0';
Willy Tarreau03938182010-03-17 21:52:07 +0100889 else {
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200890 s->check.bi->data[s->check.bi->i - 1] = '\0';
Willy Tarreau03938182010-03-17 21:52:07 +0100891 done = 1; /* buffer full, don't wait for more data */
892 }
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200893
Nick Chalk57b1bf72010-03-16 15:50:46 +0000894 /* Run the checks... */
Willy Tarreau1620ec32011-08-06 17:05:02 +0200895 switch (s->proxy->options2 & PR_O2_CHK_ANY) {
896 case PR_O2_HTTP_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200897 if (!done && s->check.bi->i < strlen("HTTP/1.0 000\r"))
Willy Tarreau03938182010-03-17 21:52:07 +0100898 goto wait_more_data;
899
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100900 /* Check if the server speaks HTTP 1.X */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200901 if ((s->check.bi->i < strlen("HTTP/1.0 000\r")) ||
902 (memcmp(s->check.bi->data, "HTTP/1.", 7) != 0 ||
903 (*(s->check.bi->data + 12) != ' ' && *(s->check.bi->data + 12) != '\r')) ||
904 !isdigit((unsigned char) *(s->check.bi->data + 9)) || !isdigit((unsigned char) *(s->check.bi->data + 10)) ||
905 !isdigit((unsigned char) *(s->check.bi->data + 11))) {
906 cut_crlf(s->check.bi->data);
907 set_server_check_status(s, HCHK_STATUS_L7RSP, s->check.bi->data);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200908
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100909 goto out_wakeup;
910 }
911
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200912 s->check.code = str2uic(s->check.bi->data + 9);
913 desc = ltrim(s->check.bi->data + 12, ' ');
Nick Chalk57b1bf72010-03-16 15:50:46 +0000914
Willy Tarreaubd741542010-03-16 18:46:54 +0100915 if ((s->proxy->options & PR_O_DISABLE404) &&
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200916 (s->state & SRV_RUNNING) && (s->check.code == 404)) {
Nick Chalk57b1bf72010-03-16 15:50:46 +0000917 /* 404 may be accepted as "stopping" only if the server was up */
918 cut_crlf(desc);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200919 set_server_check_status(s, HCHK_STATUS_L7OKCD, desc);
Nick Chalk57b1bf72010-03-16 15:50:46 +0000920 }
Willy Tarreaubd741542010-03-16 18:46:54 +0100921 else if (s->proxy->options2 & PR_O2_EXP_TYPE) {
922 /* Run content verification check... We know we have at least 13 chars */
923 if (!httpchk_expect(s, done))
924 goto wait_more_data;
925 }
926 /* check the reply : HTTP/1.X 2xx and 3xx are OK */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200927 else if (*(s->check.bi->data + 9) == '2' || *(s->check.bi->data + 9) == '3') {
Willy Tarreaubd741542010-03-16 18:46:54 +0100928 cut_crlf(desc);
929 set_server_check_status(s, HCHK_STATUS_L7OKD, desc);
930 }
Nick Chalk57b1bf72010-03-16 15:50:46 +0000931 else {
932 cut_crlf(desc);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200933 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
Nick Chalk57b1bf72010-03-16 15:50:46 +0000934 }
Willy Tarreau1620ec32011-08-06 17:05:02 +0200935 break;
936
937 case PR_O2_SSL3_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200938 if (!done && s->check.bi->i < 5)
Willy Tarreau03938182010-03-17 21:52:07 +0100939 goto wait_more_data;
940
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100941 /* Check for SSLv3 alert or handshake */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200942 if ((s->check.bi->i >= 5) && (*s->check.bi->data == 0x15 || *s->check.bi->data == 0x16))
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200943 set_server_check_status(s, HCHK_STATUS_L6OK, NULL);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200944 else
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200945 set_server_check_status(s, HCHK_STATUS_L6RSP, NULL);
Willy Tarreau1620ec32011-08-06 17:05:02 +0200946 break;
947
948 case PR_O2_SMTP_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200949 if (!done && s->check.bi->i < strlen("000\r"))
Willy Tarreau03938182010-03-17 21:52:07 +0100950 goto wait_more_data;
951
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200952 /* Check if the server speaks SMTP */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200953 if ((s->check.bi->i < strlen("000\r")) ||
954 (*(s->check.bi->data + 3) != ' ' && *(s->check.bi->data + 3) != '\r') ||
955 !isdigit((unsigned char) *s->check.bi->data) || !isdigit((unsigned char) *(s->check.bi->data + 1)) ||
956 !isdigit((unsigned char) *(s->check.bi->data + 2))) {
957 cut_crlf(s->check.bi->data);
958 set_server_check_status(s, HCHK_STATUS_L7RSP, s->check.bi->data);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200959
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200960 goto out_wakeup;
961 }
962
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200963 s->check.code = str2uic(s->check.bi->data);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200964
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200965 desc = ltrim(s->check.bi->data + 3, ' ');
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200966 cut_crlf(desc);
967
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100968 /* Check for SMTP code 2xx (should be 250) */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200969 if (*s->check.bi->data == '2')
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200970 set_server_check_status(s, HCHK_STATUS_L7OKD, desc);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200971 else
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200972 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
Willy Tarreau1620ec32011-08-06 17:05:02 +0200973 break;
974
975 case PR_O2_PGSQL_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200976 if (!done && s->check.bi->i < 9)
Rauf Kuliyev38b41562011-01-04 15:14:13 +0100977 goto wait_more_data;
978
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200979 if (s->check.bi->data[0] == 'R') {
Rauf Kuliyev38b41562011-01-04 15:14:13 +0100980 set_server_check_status(s, HCHK_STATUS_L7OKD, "PostgreSQL server is ok");
981 }
982 else {
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200983 if ((s->check.bi->data[0] == 'E') && (s->check.bi->data[5]!=0) && (s->check.bi->data[6]!=0))
984 desc = &s->check.bi->data[6];
Rauf Kuliyev38b41562011-01-04 15:14:13 +0100985 else
986 desc = "PostgreSQL unknown error";
987
988 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
989 }
Willy Tarreau1620ec32011-08-06 17:05:02 +0200990 break;
991
992 case PR_O2_REDIS_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200993 if (!done && s->check.bi->i < 7)
Hervé COMMOWICKec032d62011-08-05 16:23:48 +0200994 goto wait_more_data;
995
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200996 if (strcmp(s->check.bi->data, "+PONG\r\n") == 0) {
Hervé COMMOWICKec032d62011-08-05 16:23:48 +0200997 set_server_check_status(s, HCHK_STATUS_L7OKD, "Redis server is ok");
998 }
999 else {
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001000 set_server_check_status(s, HCHK_STATUS_L7STS, s->check.bi->data);
Hervé COMMOWICKec032d62011-08-05 16:23:48 +02001001 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001002 break;
1003
1004 case PR_O2_MYSQL_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001005 if (!done && s->check.bi->i < 5)
Willy Tarreau03938182010-03-17 21:52:07 +01001006 goto wait_more_data;
1007
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001008 if (s->proxy->check_len == 0) { // old mode
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001009 if (*(s->check.bi->data + 4) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001010 /* We set the MySQL Version in description for information purpose
1011 * FIXME : it can be cool to use MySQL Version for other purpose,
1012 * like mark as down old MySQL server.
1013 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001014 if (s->check.bi->i > 51) {
1015 desc = ltrim(s->check.bi->data + 5, ' ');
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001016 set_server_check_status(s, HCHK_STATUS_L7OKD, desc);
1017 }
1018 else {
1019 if (!done)
1020 goto wait_more_data;
1021 /* it seems we have a OK packet but without a valid length,
1022 * it must be a protocol error
1023 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001024 set_server_check_status(s, HCHK_STATUS_L7RSP, s->check.bi->data);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001025 }
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001026 }
1027 else {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001028 /* An error message is attached in the Error packet */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001029 desc = ltrim(s->check.bi->data + 7, ' ');
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001030 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
1031 }
1032 } else {
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001033 unsigned int first_packet_len = ((unsigned int) *s->check.bi->data) +
1034 (((unsigned int) *(s->check.bi->data + 1)) << 8) +
1035 (((unsigned int) *(s->check.bi->data + 2)) << 16);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001036
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001037 if (s->check.bi->i == first_packet_len + 4) {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001038 /* MySQL Error packet always begin with field_count = 0xff */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001039 if (*(s->check.bi->data + 4) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001040 /* We have only one MySQL packet and it is a Handshake Initialization packet
1041 * but we need to have a second packet to know if it is alright
1042 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001043 if (!done && s->check.bi->i < first_packet_len + 5)
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001044 goto wait_more_data;
1045 }
1046 else {
1047 /* We have only one packet and it is an Error packet,
1048 * an error message is attached, so we can display it
1049 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001050 desc = &s->check.bi->data[7];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001051 //Warning("onlyoneERR: %s\n", desc);
1052 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
1053 }
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001054 } else if (s->check.bi->i > first_packet_len + 4) {
1055 unsigned int second_packet_len = ((unsigned int) *(s->check.bi->data + first_packet_len + 4)) +
1056 (((unsigned int) *(s->check.bi->data + first_packet_len + 5)) << 8) +
1057 (((unsigned int) *(s->check.bi->data + first_packet_len + 6)) << 16);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001058
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001059 if (s->check.bi->i == first_packet_len + 4 + second_packet_len + 4 ) {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001060 /* We have 2 packets and that's good */
1061 /* Check if the second packet is a MySQL Error packet or not */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001062 if (*(s->check.bi->data + first_packet_len + 8) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001063 /* No error packet */
1064 /* We set the MySQL Version in description for information purpose */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001065 desc = &s->check.bi->data[5];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001066 //Warning("2packetOK: %s\n", desc);
1067 set_server_check_status(s, HCHK_STATUS_L7OKD, desc);
1068 }
1069 else {
1070 /* An error message is attached in the Error packet
1071 * so we can display it ! :)
1072 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001073 desc = &s->check.bi->data[first_packet_len+11];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001074 //Warning("2packetERR: %s\n", desc);
1075 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
1076 }
1077 }
1078 }
1079 else {
Willy Tarreau03938182010-03-17 21:52:07 +01001080 if (!done)
1081 goto wait_more_data;
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001082 /* it seems we have a Handshake Initialization packet but without a valid length,
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001083 * it must be a protocol error
1084 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001085 desc = &s->check.bi->data[5];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001086 //Warning("protoerr: %s\n", desc);
1087 set_server_check_status(s, HCHK_STATUS_L7RSP, desc);
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001088 }
1089 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001090 break;
1091
1092 case PR_O2_LDAP_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001093 if (!done && s->check.bi->i < 14)
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001094 goto wait_more_data;
1095
1096 /* Check if the server speaks LDAP (ASN.1/BER)
1097 * http://en.wikipedia.org/wiki/Basic_Encoding_Rules
1098 * http://tools.ietf.org/html/rfc4511
1099 */
1100
1101 /* http://tools.ietf.org/html/rfc4511#section-4.1.1
1102 * LDAPMessage: 0x30: SEQUENCE
1103 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001104 if ((s->check.bi->i < 14) || (*(s->check.bi->data) != '\x30')) {
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001105 set_server_check_status(s, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
1106 }
1107 else {
1108 /* size of LDAPMessage */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001109 msglen = (*(s->check.bi->data + 1) & 0x80) ? (*(s->check.bi->data + 1) & 0x7f) : 0;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001110
1111 /* http://tools.ietf.org/html/rfc4511#section-4.2.2
1112 * messageID: 0x02 0x01 0x01: INTEGER 1
1113 * protocolOp: 0x61: bindResponse
1114 */
1115 if ((msglen > 2) ||
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001116 (memcmp(s->check.bi->data + 2 + msglen, "\x02\x01\x01\x61", 4) != 0)) {
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001117 set_server_check_status(s, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
1118
1119 goto out_wakeup;
1120 }
1121
1122 /* size of bindResponse */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001123 msglen += (*(s->check.bi->data + msglen + 6) & 0x80) ? (*(s->check.bi->data + msglen + 6) & 0x7f) : 0;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001124
1125 /* http://tools.ietf.org/html/rfc4511#section-4.1.9
1126 * ldapResult: 0x0a 0x01: ENUMERATION
1127 */
1128 if ((msglen > 4) ||
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001129 (memcmp(s->check.bi->data + 7 + msglen, "\x0a\x01", 2) != 0)) {
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001130 set_server_check_status(s, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
1131
1132 goto out_wakeup;
1133 }
1134
1135 /* http://tools.ietf.org/html/rfc4511#section-4.1.9
1136 * resultCode
1137 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001138 s->check.code = *(s->check.bi->data + msglen + 9);
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001139 if (s->check.code) {
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001140 set_server_check_status(s, HCHK_STATUS_L7STS, "See RFC: http://tools.ietf.org/html/rfc4511#section-4.1.9");
1141 } else {
1142 set_server_check_status(s, HCHK_STATUS_L7OKD, "Success");
1143 }
1144 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001145 break;
1146
1147 default:
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001148 /* other checks are valid if the connection succeeded anyway */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001149 set_server_check_status(s, HCHK_STATUS_L4OK, NULL);
Willy Tarreau1620ec32011-08-06 17:05:02 +02001150 break;
1151 } /* switch */
Willy Tarreau83749182007-04-15 20:56:27 +02001152
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001153 out_wakeup:
1154 if (s->result & SRV_CHK_ERROR)
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001155 conn->flags |= CO_FL_ERROR;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001156
Nick Chalk57b1bf72010-03-16 15:50:46 +00001157 /* Reset the check buffer... */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001158 *s->check.bi->data = '\0';
1159 s->check.bi->i = 0;
Nick Chalk57b1bf72010-03-16 15:50:46 +00001160
Willy Tarreau03938182010-03-17 21:52:07 +01001161 /* Close the connection... */
Willy Tarreauf1503172012-09-28 19:39:36 +02001162 if (conn->xprt && conn->xprt->shutw)
1163 conn->xprt->shutw(conn, 0);
1164 if (!(conn->flags & (CO_FL_WAIT_L4_CONN|CO_FL_SOCK_WR_SH)))
1165 shutdown(conn->t.sock.fd, SHUT_RDWR);
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001166 __conn_data_stop_recv(conn);
Willy Tarreaufdccded2008-08-29 18:19:04 +02001167 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreau3267d362012-08-17 23:53:56 +02001168 return;
Willy Tarreau03938182010-03-17 21:52:07 +01001169
1170 wait_more_data:
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001171 __conn_data_poll_recv(conn);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001172}
1173
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001174/*
1175 * This function is used only for server health-checks. It handles connection
1176 * status updates including errors. If necessary, it wakes the check task up.
1177 * It always returns 0.
1178 */
1179static int wake_srv_chk(struct connection *conn)
Willy Tarreau20bea422012-07-06 12:00:49 +02001180{
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001181 struct server *s = conn->owner;
Willy Tarreau20bea422012-07-06 12:00:49 +02001182
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001183 if (unlikely(conn->flags & CO_FL_ERROR))
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001184 task_wakeup(s->check.task, TASK_WOKEN_IO);
Willy Tarreau20bea422012-07-06 12:00:49 +02001185
Willy Tarreau3267d362012-08-17 23:53:56 +02001186 return 0;
Willy Tarreau20bea422012-07-06 12:00:49 +02001187}
1188
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001189struct data_cb check_conn_cb = {
1190 .recv = event_srv_chk_r,
1191 .send = event_srv_chk_w,
1192 .wake = wake_srv_chk,
1193};
1194
Willy Tarreaubaaee002006-06-26 02:48:02 +02001195/*
Willy Tarreau2e993902011-10-31 11:53:20 +01001196 * updates the server's weight during a warmup stage. Once the final weight is
1197 * reached, the task automatically stops. Note that any server status change
1198 * must have updated s->last_change accordingly.
1199 */
1200static struct task *server_warmup(struct task *t)
1201{
1202 struct server *s = t->context;
1203
1204 /* by default, plan on stopping the task */
1205 t->expire = TICK_ETERNITY;
1206 if ((s->state & (SRV_RUNNING|SRV_WARMINGUP|SRV_MAINTAIN)) != (SRV_RUNNING|SRV_WARMINGUP))
1207 return t;
1208
1209 if (now.tv_sec < s->last_change || now.tv_sec >= s->last_change + s->slowstart) {
1210 /* go to full throttle if the slowstart interval is reached */
1211 s->state &= ~SRV_WARMINGUP;
1212 if (s->proxy->lbprm.algo & BE_LB_PROP_DYN)
1213 s->eweight = s->uweight * BE_WEIGHT_SCALE;
1214 if (s->proxy->lbprm.update_server_eweight)
1215 s->proxy->lbprm.update_server_eweight(s);
1216 }
1217 else if (s->proxy->lbprm.algo & BE_LB_PROP_DYN) {
1218 /* for dynamic algorithms, let's slowly update the weight */
1219 s->eweight = (BE_WEIGHT_SCALE * (now.tv_sec - s->last_change) +
1220 s->slowstart - 1) / s->slowstart;
1221 s->eweight *= s->uweight;
1222 if (s->proxy->lbprm.update_server_eweight)
1223 s->proxy->lbprm.update_server_eweight(s);
1224 }
1225 /* Note that static algorithms are already running at full throttle */
1226
1227 /* probably that we can refill this server with a bit more connections */
1228 check_for_pending(s);
1229
1230 /* get back there in 1 second or 1/20th of the slowstart interval,
1231 * whichever is greater, resulting in small 5% steps.
1232 */
1233 if (s->state & SRV_WARMINGUP)
1234 t->expire = tick_add(now_ms, MS_TO_TICKS(MAX(1000, s->slowstart / 20)));
1235 return t;
1236}
1237
1238/*
Willy Tarreaubaaee002006-06-26 02:48:02 +02001239 * manages a server health-check. Returns
1240 * the time the task accepts to wait, or TIME_ETERNITY for infinity.
1241 */
Simon Horman63a4a822012-03-19 07:24:41 +09001242static struct task *process_chk(struct task *t)
Willy Tarreaubaaee002006-06-26 02:48:02 +02001243{
Willy Tarreaue3838802009-03-21 18:58:32 +01001244 int attempts = 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001245 struct server *s = t->context;
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001246 struct connection *conn = s->check.conn;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001247 int fd;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001248 int rv;
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001249 int ret;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001250
Willy Tarreaubaaee002006-06-26 02:48:02 +02001251 new_chk:
Willy Tarreaue3838802009-03-21 18:58:32 +01001252 if (attempts++ > 0) {
1253 /* we always fail to create a server, let's stop insisting... */
1254 while (tick_is_expired(t->expire, now_ms))
1255 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
1256 return t;
1257 }
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001258
1259 fd = conn->t.sock.fd;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001260 if (fd < 0) { /* no check currently running */
Willy Tarreau26c25062009-03-08 09:38:41 +01001261 if (!tick_is_expired(t->expire, now_ms)) /* woke up too early */
1262 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001263
1264 /* we don't send any health-checks when the proxy is stopped or when
1265 * the server should not be checked.
1266 */
Cyril Bontécd19e512010-01-31 22:34:03 +01001267 if (!(s->state & SRV_CHECKED) || s->proxy->state == PR_STSTOPPED || (s->state & SRV_MAINTAIN)) {
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001268 while (tick_is_expired(t->expire, now_ms))
1269 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
Willy Tarreau26c25062009-03-08 09:38:41 +01001270 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001271 }
1272
1273 /* we'll initiate a new check */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001274 set_server_check_status(s, HCHK_STATUS_START, NULL);
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001275
1276 s->check.bi->p = s->check.bi->data;
1277 s->check.bi->i = 0;
1278 s->check.bo->p = s->check.bo->data;
1279 s->check.bo->o = 0;
1280
1281 /* prepare the check buffer */
1282 if (s->proxy->options2 & PR_O2_CHK_ANY) {
1283 bo_putblk(s->check.bo, s->proxy->check_req, s->proxy->check_len);
1284
1285 /* we want to check if this host replies to HTTP or SSLv3 requests
1286 * so we'll send the request, and won't wake the checker up now.
1287 */
1288 if ((s->proxy->options2 & PR_O2_CHK_ANY) == PR_O2_SSL3_CHK) {
1289 /* SSL requires that we put Unix time in the request */
1290 int gmt_time = htonl(date.tv_sec);
1291 memcpy(s->check.bo->data + 11, &gmt_time, 4);
1292 }
1293 else if ((s->proxy->options2 & PR_O2_CHK_ANY) == PR_O2_HTTP_CHK) {
1294 if (s->proxy->options2 & PR_O2_CHK_SNDST)
Willy Tarreau19d14ef2012-10-29 16:51:55 +01001295 bo_putblk(s->check.bo, trash.str, httpchk_build_status_header(s, trash.str));
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001296 bo_putstr(s->check.bo, "\r\n");
1297 *s->check.bo->p = '\0'; /* to make gdb output easier to read */
1298 }
1299 }
1300
1301 /* prepare a new connection */
Willy Tarreau3fdb3662012-11-12 00:42:33 +01001302 conn->target = &s->obj_type;
Willy Tarreauf4288ee2012-09-28 18:13:10 +02001303 conn_prepare(conn, &check_conn_cb, s->check.proto, s->check.xprt, s);
Willy Tarreauc6f4ce82009-06-10 11:09:37 +02001304
Willy Tarreau5f2877a2012-10-26 19:57:58 +02001305 /* no client address */
1306 clear_addr(&conn->addr.from);
1307
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001308 if (is_addr(&s->check.addr))
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001309 /* we'll connect to the check addr specified on the server */
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001310 conn->addr.to = s->check.addr;
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001311 else
1312 /* we'll connect to the addr on the server */
1313 conn->addr.to = s->addr;
Willy Tarreauc6f4ce82009-06-10 11:09:37 +02001314
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001315 set_host_port(&conn->addr.to, s->check.port);
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001316
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001317 /* It can return one of :
1318 * - SN_ERR_NONE if everything's OK
1319 * - SN_ERR_SRVTO if there are no more servers
1320 * - SN_ERR_SRVCL if the connection was refused by the server
1321 * - SN_ERR_PRXCOND if the connection has been limited by the proxy (maxconn)
1322 * - SN_ERR_RESOURCE if a system resource is lacking (eg: fd limits, ports, ...)
1323 * - SN_ERR_INTERNAL for any other purely internal errors
1324 * Additionnally, in the case of SN_ERR_RESOURCE, an emergency log will be emitted.
1325 */
Willy Tarreauf4288ee2012-09-28 18:13:10 +02001326 ret = s->check.proto->connect(conn, 1);
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001327 conn->flags |= CO_FL_WAKE_DATA;
Willy Tarreau6c16adc2012-10-05 00:04:16 +02001328 if (s->check.send_proxy)
1329 conn->flags |= CO_FL_LOCAL_SPROXY;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001330
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001331 switch (ret) {
1332 case SN_ERR_NONE:
1333 break;
1334 case SN_ERR_SRVTO: /* ETIMEDOUT */
1335 case SN_ERR_SRVCL: /* ECONNREFUSED, ENETUNREACH, ... */
1336 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(errno));
1337 break;
1338 case SN_ERR_PRXCOND:
1339 case SN_ERR_RESOURCE:
1340 case SN_ERR_INTERNAL:
1341 set_server_check_status(s, HCHK_STATUS_SOCKERR, NULL);
1342 break;
1343 }
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001344
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001345 if (s->result == SRV_CHK_UNKNOWN) {
1346 /* connection attempt was started */
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001347
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001348 /* we allow up to min(inter, timeout.connect) for a connection
1349 * to establish but only when timeout.check is set
1350 * as it may be to short for a full check otherwise
1351 */
1352 t->expire = tick_add(now_ms, MS_TO_TICKS(s->inter));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001353
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001354 if (s->proxy->timeout.check && s->proxy->timeout.connect) {
1355 int t_con = tick_add(now_ms, s->proxy->timeout.connect);
1356 t->expire = tick_first(t->expire, t_con);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001357 }
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001358 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001359 }
1360
1361 /* here, we have seen a failure */
1362 if (s->health > s->rise) {
1363 s->health--; /* still good */
Krzysztof Piotr Oledzki052d4fd2009-10-04 14:52:57 +02001364 s->counters.failed_checks++;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001365 }
1366 else
1367 set_server_down(s);
1368
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001369 //fprintf(stderr, "process_chk: 7, %lu\n", __tv_to_ms(&s->proxy->timeout.connect));
1370 /* we allow up to min(inter, timeout.connect) for a connection
1371 * to establish but only when timeout.check is set
1372 * as it may be to short for a full check otherwise
1373 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001374 while (tick_is_expired(t->expire, now_ms)) {
1375 int t_con;
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001376
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001377 t_con = tick_add(t->expire, s->proxy->timeout.connect);
1378 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001379
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001380 if (s->proxy->timeout.check)
1381 t->expire = tick_first(t->expire, t_con);
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001382 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001383 goto new_chk;
1384 }
1385 else {
Willy Tarreauf1503172012-09-28 19:39:36 +02001386 /* there was a test running.
1387 * First, let's check whether there was an uncaught error,
1388 * which can happen on connect timeout or error.
1389 */
1390 if (s->result == SRV_CHK_UNKNOWN) {
1391 if (conn->flags & CO_FL_CONNECTED) {
1392 /* good TCP connection is enough */
1393 if (s->check.use_ssl)
1394 set_server_check_status(s, HCHK_STATUS_L6OK, NULL);
1395 else
1396 set_server_check_status(s, HCHK_STATUS_L4OK, NULL);
1397 }
1398 else if (conn->flags & CO_FL_WAIT_L4_CONN) {
1399 /* L4 failed */
1400 if (conn->flags & CO_FL_ERROR)
1401 set_server_check_status(s, HCHK_STATUS_L4CON, NULL);
1402 else
1403 set_server_check_status(s, HCHK_STATUS_L4TOUT, NULL);
1404 }
1405 else if (conn->flags & CO_FL_WAIT_L6_CONN) {
1406 /* L6 failed */
1407 if (conn->flags & CO_FL_ERROR)
1408 set_server_check_status(s, HCHK_STATUS_L6RSP, NULL);
1409 else
1410 set_server_check_status(s, HCHK_STATUS_L6TOUT, NULL);
1411 }
1412 }
1413
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001414 if ((s->result & (SRV_CHK_ERROR|SRV_CHK_RUNNING)) == SRV_CHK_RUNNING) { /* good server detected */
Willy Tarreau48494c02007-11-30 10:41:39 +01001415 /* we may have to add/remove this server from the LB group */
1416 if ((s->state & SRV_RUNNING) && (s->proxy->options & PR_O_DISABLE404)) {
1417 if ((s->state & SRV_GOINGDOWN) &&
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +01001418 ((s->result & (SRV_CHK_RUNNING|SRV_CHK_DISABLE)) == SRV_CHK_RUNNING))
1419 set_server_enabled(s);
Willy Tarreau48494c02007-11-30 10:41:39 +01001420 else if (!(s->state & SRV_GOINGDOWN) &&
1421 ((s->result & (SRV_CHK_RUNNING | SRV_CHK_DISABLE)) ==
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +01001422 (SRV_CHK_RUNNING | SRV_CHK_DISABLE)))
1423 set_server_disabled(s);
Willy Tarreau48494c02007-11-30 10:41:39 +01001424 }
1425
Krzysztof Oledzki85130942007-10-22 16:21:10 +02001426 if (s->health < s->rise + s->fall - 1) {
1427 s->health++; /* was bad, stays for a while */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001428
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +01001429 set_server_up(s);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001430 }
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001431 conn->t.sock.fd = -1; /* no check running anymore */
1432 conn_xprt_close(conn);
1433 if (conn->ctrl)
1434 fd_delete(fd);
Willy Tarreau44ec0f02007-10-14 23:47:04 +02001435
1436 rv = 0;
1437 if (global.spread_checks > 0) {
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001438 rv = srv_getinter(s) * global.spread_checks / 100;
Willy Tarreau44ec0f02007-10-14 23:47:04 +02001439 rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0)));
Willy Tarreau44ec0f02007-10-14 23:47:04 +02001440 }
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001441 t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(s) + rv));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001442 goto new_chk;
1443 }
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001444 else if ((s->result & SRV_CHK_ERROR) || tick_is_expired(t->expire, now_ms)) {
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001445 if (!(s->result & SRV_CHK_ERROR)) {
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001446 if (conn->flags & CO_FL_WAIT_L4_CONN) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001447 set_server_check_status(s, HCHK_STATUS_L4TOUT, NULL);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001448 } else {
Willy Tarreau1620ec32011-08-06 17:05:02 +02001449 if ((s->proxy->options2 & PR_O2_CHK_ANY) == PR_O2_SSL3_CHK)
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001450 set_server_check_status(s, HCHK_STATUS_L6TOUT, NULL);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001451 else /* HTTP, SMTP */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001452 set_server_check_status(s, HCHK_STATUS_L7TOUT, NULL);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001453 }
1454 }
1455
Willy Tarreaubaaee002006-06-26 02:48:02 +02001456 /* failure or timeout detected */
1457 if (s->health > s->rise) {
1458 s->health--; /* still good */
Krzysztof Piotr Oledzki052d4fd2009-10-04 14:52:57 +02001459 s->counters.failed_checks++;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001460 }
1461 else
1462 set_server_down(s);
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001463 conn->t.sock.fd = -1;
1464 conn_xprt_close(conn);
1465 if (conn->ctrl)
1466 fd_delete(fd);
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001467
1468 rv = 0;
1469 if (global.spread_checks > 0) {
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001470 rv = srv_getinter(s) * global.spread_checks / 100;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001471 rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0)));
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001472 }
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001473 t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(s) + rv));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001474 goto new_chk;
1475 }
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001476 /* if result is unknown and there's no timeout, we have to wait again */
Willy Tarreaubaaee002006-06-26 02:48:02 +02001477 }
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001478 s->result = SRV_CHK_UNKNOWN;
Willy Tarreau26c25062009-03-08 09:38:41 +01001479 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001480}
1481
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001482/*
1483 * Start health-check.
1484 * Returns 0 if OK, -1 if error, and prints the error in this case.
1485 */
1486int start_checks() {
1487
1488 struct proxy *px;
1489 struct server *s;
1490 struct task *t;
1491 int nbchk=0, mininter=0, srvpos=0;
1492
Willy Tarreau2c43a1e2007-10-14 23:05:39 +02001493 /* 1- count the checkers to run simultaneously.
1494 * We also determine the minimum interval among all of those which
1495 * have an interval larger than SRV_CHK_INTER_THRES. This interval
1496 * will be used to spread their start-up date. Those which have
Jamie Gloudon801a0a32012-08-25 00:18:33 -04001497 * a shorter interval will start independently and will not dictate
Willy Tarreau2c43a1e2007-10-14 23:05:39 +02001498 * too short an interval for all others.
1499 */
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001500 for (px = proxy; px; px = px->next) {
1501 for (s = px->srv; s; s = s->next) {
1502 if (!(s->state & SRV_CHECKED))
1503 continue;
1504
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001505 if ((srv_getinter(s) >= SRV_CHK_INTER_THRES) &&
1506 (!mininter || mininter > srv_getinter(s)))
1507 mininter = srv_getinter(s);
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001508
1509 nbchk++;
1510 }
1511 }
1512
1513 if (!nbchk)
1514 return 0;
1515
1516 srand((unsigned)time(NULL));
1517
1518 /*
1519 * 2- start them as far as possible from each others. For this, we will
1520 * start them after their interval set to the min interval divided by
1521 * the number of servers, weighted by the server's position in the list.
1522 */
1523 for (px = proxy; px; px = px->next) {
1524 for (s = px->srv; s; s = s->next) {
Willy Tarreau2e993902011-10-31 11:53:20 +01001525 if (s->slowstart) {
1526 if ((t = task_new()) == NULL) {
1527 Alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
1528 return -1;
1529 }
1530 /* We need a warmup task that will be called when the server
1531 * state switches from down to up.
1532 */
1533 s->warmup = t;
1534 t->process = server_warmup;
1535 t->context = s;
1536 t->expire = TICK_ETERNITY;
1537 }
1538
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001539 if (!(s->state & SRV_CHECKED))
1540 continue;
1541
Willy Tarreau2e993902011-10-31 11:53:20 +01001542 /* one task for the checks */
Willy Tarreaua4613182009-03-21 18:13:21 +01001543 if ((t = task_new()) == NULL) {
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001544 Alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
1545 return -1;
1546 }
1547
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001548 s->check.task = t;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001549 t->process = process_chk;
1550 t->context = s;
1551
1552 /* check this every ms */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001553 t->expire = tick_add(now_ms,
1554 MS_TO_TICKS(((mininter && mininter >= srv_getinter(s)) ?
1555 mininter : srv_getinter(s)) * srvpos / nbchk));
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001556 s->check.start = now;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001557 task_queue(t);
1558
1559 srvpos++;
1560 }
1561 }
1562 return 0;
1563}
Willy Tarreaubaaee002006-06-26 02:48:02 +02001564
1565/*
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001566 * Perform content verification check on data in s->check.buffer buffer.
Willy Tarreaubd741542010-03-16 18:46:54 +01001567 * The buffer MUST be terminated by a null byte before calling this function.
1568 * Sets server status appropriately. The caller is responsible for ensuring
1569 * that the buffer contains at least 13 characters. If <done> is zero, we may
1570 * return 0 to indicate that data is required to decide of a match.
1571 */
1572static int httpchk_expect(struct server *s, int done)
1573{
1574 static char status_msg[] = "HTTP status check returned code <000>";
1575 char status_code[] = "000";
1576 char *contentptr;
1577 int crlf;
1578 int ret;
1579
1580 switch (s->proxy->options2 & PR_O2_EXP_TYPE) {
1581 case PR_O2_EXP_STS:
1582 case PR_O2_EXP_RSTS:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001583 memcpy(status_code, s->check.bi->data + 9, 3);
1584 memcpy(status_msg + strlen(status_msg) - 4, s->check.bi->data + 9, 3);
Willy Tarreaubd741542010-03-16 18:46:54 +01001585
1586 if ((s->proxy->options2 & PR_O2_EXP_TYPE) == PR_O2_EXP_STS)
1587 ret = strncmp(s->proxy->expect_str, status_code, 3) == 0;
1588 else
1589 ret = regexec(s->proxy->expect_regex, status_code, MAX_MATCH, pmatch, 0) == 0;
1590
1591 /* we necessarily have the response, so there are no partial failures */
1592 if (s->proxy->options2 & PR_O2_EXP_INV)
1593 ret = !ret;
1594
1595 set_server_check_status(s, ret ? HCHK_STATUS_L7OKD : HCHK_STATUS_L7STS, status_msg);
1596 break;
1597
1598 case PR_O2_EXP_STR:
1599 case PR_O2_EXP_RSTR:
1600 /* very simple response parser: ignore CR and only count consecutive LFs,
1601 * stop with contentptr pointing to first char after the double CRLF or
1602 * to '\0' if crlf < 2.
1603 */
1604 crlf = 0;
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001605 for (contentptr = s->check.bi->data; *contentptr; contentptr++) {
Willy Tarreaubd741542010-03-16 18:46:54 +01001606 if (crlf >= 2)
1607 break;
1608 if (*contentptr == '\r')
1609 continue;
1610 else if (*contentptr == '\n')
1611 crlf++;
1612 else
1613 crlf = 0;
1614 }
1615
1616 /* Check that response contains a body... */
1617 if (crlf < 2) {
1618 if (!done)
1619 return 0;
1620
1621 set_server_check_status(s, HCHK_STATUS_L7RSP,
1622 "HTTP content check could not find a response body");
1623 return 1;
1624 }
1625
1626 /* Check that response body is not empty... */
1627 if (*contentptr == '\0') {
Willy Tarreaua164fb52011-04-13 09:32:41 +02001628 if (!done)
1629 return 0;
1630
Willy Tarreaubd741542010-03-16 18:46:54 +01001631 set_server_check_status(s, HCHK_STATUS_L7RSP,
1632 "HTTP content check found empty response body");
1633 return 1;
1634 }
1635
1636 /* Check the response content against the supplied string
1637 * or regex... */
1638 if ((s->proxy->options2 & PR_O2_EXP_TYPE) == PR_O2_EXP_STR)
1639 ret = strstr(contentptr, s->proxy->expect_str) != NULL;
1640 else
1641 ret = regexec(s->proxy->expect_regex, contentptr, MAX_MATCH, pmatch, 0) == 0;
1642
1643 /* if we don't match, we may need to wait more */
1644 if (!ret && !done)
1645 return 0;
1646
1647 if (ret) {
1648 /* content matched */
1649 if (s->proxy->options2 & PR_O2_EXP_INV)
1650 set_server_check_status(s, HCHK_STATUS_L7RSP,
1651 "HTTP check matched unwanted content");
1652 else
1653 set_server_check_status(s, HCHK_STATUS_L7OKD,
1654 "HTTP content check matched");
1655 }
1656 else {
1657 if (s->proxy->options2 & PR_O2_EXP_INV)
1658 set_server_check_status(s, HCHK_STATUS_L7OKD,
1659 "HTTP check did not match unwanted content");
1660 else
1661 set_server_check_status(s, HCHK_STATUS_L7RSP,
1662 "HTTP content check did not match");
1663 }
1664 break;
1665 }
1666 return 1;
1667}
1668
1669/*
Willy Tarreaubaaee002006-06-26 02:48:02 +02001670 * Local variables:
1671 * c-indent-level: 8
1672 * c-basic-offset: 8
1673 * End:
1674 */