blob: 394d29ef75be1936e4b498d2e0eceb6e70473277 [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
2 * Health-checks functions.
3 *
Willy Tarreau26c25062009-03-08 09:38:41 +01004 * Copyright 2000-2009 Willy Tarreau <w@1wt.eu>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02005 * Copyright 2007-2009 Krzysztof Piotr Oledzki <ole@ans.pl>
Willy Tarreaubaaee002006-06-26 02:48:02 +02006 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 *
12 */
13
Willy Tarreaub8816082008-01-18 12:18:15 +010014#include <assert.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020015#include <ctype.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020016#include <errno.h>
17#include <fcntl.h>
18#include <stdio.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020019#include <stdlib.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020020#include <string.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020021#include <time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020022#include <unistd.h>
23#include <sys/socket.h>
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +040024#include <sys/types.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020025#include <netinet/in.h>
Willy Tarreau1274bc42009-07-15 07:16:31 +020026#include <netinet/tcp.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020027#include <arpa/inet.h>
28
Willy Tarreauc7e42382012-08-24 19:22:53 +020029#include <common/chunk.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020030#include <common/compat.h>
31#include <common/config.h>
32#include <common/mini-clist.h>
Willy Tarreau83749182007-04-15 20:56:27 +020033#include <common/standard.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020034#include <common/time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020035
36#include <types/global.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020037
38#include <proto/backend.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020039#include <proto/checks.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020040#include <proto/fd.h>
41#include <proto/log.h>
42#include <proto/queue.h>
Willy Tarreauc6f4ce82009-06-10 11:09:37 +020043#include <proto/port_range.h>
Willy Tarreau3d300592007-03-18 18:34:41 +010044#include <proto/proto_http.h>
Willy Tarreaue8c66af2008-01-13 18:40:14 +010045#include <proto/proto_tcp.h>
Willy Tarreau2b5652f2006-12-31 17:46:05 +010046#include <proto/proxy.h>
Willy Tarreaufb56aab2012-09-28 14:40:02 +020047#include <proto/raw_sock.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020048#include <proto/server.h>
Simon Hormane0d1bfb2011-06-21 14:34:58 +090049#include <proto/session.h>
Willy Tarreau9e000c62011-03-10 14:03:36 +010050#include <proto/stream_interface.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020051#include <proto/task.h>
52
Willy Tarreaubd741542010-03-16 18:46:54 +010053static int httpchk_expect(struct server *s, int done);
54
Simon Horman63a4a822012-03-19 07:24:41 +090055static const struct check_status check_statuses[HCHK_STATUS_SIZE] = {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020056 [HCHK_STATUS_UNKNOWN] = { SRV_CHK_UNKNOWN, "UNK", "Unknown" },
57 [HCHK_STATUS_INI] = { SRV_CHK_UNKNOWN, "INI", "Initializing" },
58 [HCHK_STATUS_START] = { /* SPECIAL STATUS*/ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020059
Willy Tarreaud3aac702012-11-23 11:32:12 +010060 [HCHK_STATUS_HANA] = { SRV_CHK_FAILED, "HANA", "Health analyze" },
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010061
Willy Tarreaud3aac702012-11-23 11:32:12 +010062 [HCHK_STATUS_SOCKERR] = { SRV_CHK_FAILED, "SOCKERR", "Socket error" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020063
Willy Tarreaud3aac702012-11-23 11:32:12 +010064 [HCHK_STATUS_L4OK] = { SRV_CHK_PASSED, "L4OK", "Layer4 check passed" },
65 [HCHK_STATUS_L4TOUT] = { SRV_CHK_FAILED, "L4TOUT", "Layer4 timeout" },
66 [HCHK_STATUS_L4CON] = { SRV_CHK_FAILED, "L4CON", "Layer4 connection problem" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020067
Willy Tarreaud3aac702012-11-23 11:32:12 +010068 [HCHK_STATUS_L6OK] = { SRV_CHK_PASSED, "L6OK", "Layer6 check passed" },
69 [HCHK_STATUS_L6TOUT] = { SRV_CHK_FAILED, "L6TOUT", "Layer6 timeout" },
70 [HCHK_STATUS_L6RSP] = { SRV_CHK_FAILED, "L6RSP", "Layer6 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020071
Willy Tarreaud3aac702012-11-23 11:32:12 +010072 [HCHK_STATUS_L7TOUT] = { SRV_CHK_FAILED, "L7TOUT", "Layer7 timeout" },
73 [HCHK_STATUS_L7RSP] = { SRV_CHK_FAILED, "L7RSP", "Layer7 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020074
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020075 [HCHK_STATUS_L57DATA] = { /* DUMMY STATUS */ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020076
Willy Tarreaud3aac702012-11-23 11:32:12 +010077 [HCHK_STATUS_L7OKD] = { SRV_CHK_PASSED, "L7OK", "Layer7 check passed" },
78 [HCHK_STATUS_L7OKCD] = { SRV_CHK_PASSED | SRV_CHK_DISABLE, "L7OKC", "Layer7 check conditionally passed" },
79 [HCHK_STATUS_L7STS] = { SRV_CHK_FAILED, "L7STS", "Layer7 wrong status" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020080};
81
Simon Horman63a4a822012-03-19 07:24:41 +090082static const struct analyze_status analyze_statuses[HANA_STATUS_SIZE] = { /* 0: ignore, 1: error, 2: OK */
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010083 [HANA_STATUS_UNKNOWN] = { "Unknown", { 0, 0 }},
84
85 [HANA_STATUS_L4_OK] = { "L4 successful connection", { 2, 0 }},
86 [HANA_STATUS_L4_ERR] = { "L4 unsuccessful connection", { 1, 1 }},
87
88 [HANA_STATUS_HTTP_OK] = { "Correct http response", { 0, 2 }},
89 [HANA_STATUS_HTTP_STS] = { "Wrong http response", { 0, 1 }},
90 [HANA_STATUS_HTTP_HDRRSP] = { "Invalid http response (headers)", { 0, 1 }},
91 [HANA_STATUS_HTTP_RSP] = { "Invalid http response", { 0, 1 }},
92
93 [HANA_STATUS_HTTP_READ_ERROR] = { "Read error (http)", { 0, 1 }},
94 [HANA_STATUS_HTTP_READ_TIMEOUT] = { "Read timeout (http)", { 0, 1 }},
95 [HANA_STATUS_HTTP_BROKEN_PIPE] = { "Close from server (http)", { 0, 1 }},
96};
97
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020098/*
99 * Convert check_status code to description
100 */
101const char *get_check_status_description(short check_status) {
102
103 const char *desc;
104
105 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200106 desc = check_statuses[check_status].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200107 else
108 desc = NULL;
109
110 if (desc && *desc)
111 return desc;
112 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200113 return check_statuses[HCHK_STATUS_UNKNOWN].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200114}
115
116/*
117 * Convert check_status code to short info
118 */
119const char *get_check_status_info(short check_status) {
120
121 const char *info;
122
123 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200124 info = check_statuses[check_status].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200125 else
126 info = NULL;
127
128 if (info && *info)
129 return info;
130 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200131 return check_statuses[HCHK_STATUS_UNKNOWN].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200132}
133
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100134const char *get_analyze_status(short analyze_status) {
135
136 const char *desc;
137
138 if (analyze_status < HANA_STATUS_SIZE)
139 desc = analyze_statuses[analyze_status].desc;
140 else
141 desc = NULL;
142
143 if (desc && *desc)
144 return desc;
145 else
146 return analyze_statuses[HANA_STATUS_UNKNOWN].desc;
147}
148
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200149#define SSP_O_HCHK 0x0002
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200150
151static void server_status_printf(struct chunk *msg, struct server *s, unsigned options, int xferred) {
152
Willy Tarreau44267702011-10-28 15:35:33 +0200153 if (s->track)
Willy Tarreau77804732012-10-29 16:14:26 +0100154 chunk_appendf(msg, " via %s/%s",
Willy Tarreau44267702011-10-28 15:35:33 +0200155 s->track->proxy->id, s->track->id);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200156
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200157 if (options & SSP_O_HCHK) {
Willy Tarreau77804732012-10-29 16:14:26 +0100158 chunk_appendf(msg, ", reason: %s", get_check_status_description(s->check.status));
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200159
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200160 if (s->check.status >= HCHK_STATUS_L57DATA)
Willy Tarreau77804732012-10-29 16:14:26 +0100161 chunk_appendf(msg, ", code: %d", s->check.code);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200162
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200163 if (*s->check.desc) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200164 struct chunk src;
165
Willy Tarreau77804732012-10-29 16:14:26 +0100166 chunk_appendf(msg, ", info: \"");
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200167
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200168 chunk_initlen(&src, s->check.desc, 0, strlen(s->check.desc));
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200169 chunk_asciiencode(msg, &src, '"');
170
Willy Tarreau77804732012-10-29 16:14:26 +0100171 chunk_appendf(msg, "\"");
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200172 }
173
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200174 if (s->check.duration >= 0)
Willy Tarreau77804732012-10-29 16:14:26 +0100175 chunk_appendf(msg, ", check duration: %ldms", s->check.duration);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200176 }
177
Krzysztof Piotr Oledzki3bb05712010-09-27 13:10:50 +0200178 if (xferred >= 0) {
Krzysztof Piotr Oledzkib16a6072010-01-10 21:12:58 +0100179 if (!(s->state & SRV_RUNNING))
Willy Tarreau77804732012-10-29 16:14:26 +0100180 chunk_appendf(msg, ". %d active and %d backup servers left.%s"
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100181 " %d sessions active, %d requeued, %d remaining in queue",
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200182 s->proxy->srv_act, s->proxy->srv_bck,
183 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
184 s->cur_sess, xferred, s->nbpend);
185 else
Willy Tarreau77804732012-10-29 16:14:26 +0100186 chunk_appendf(msg, ". %d active and %d backup servers online.%s"
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100187 " %d sessions requeued, %d total in queue",
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200188 s->proxy->srv_act, s->proxy->srv_bck,
189 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
190 xferred, s->nbpend);
191 }
192}
193
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200194/*
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200195 * Set s->check.status, update s->check.duration and fill s->result with
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200196 * an adequate SRV_CHK_* value.
197 *
198 * Show information in logs about failed health check if server is UP
199 * or succeeded health checks if server is DOWN.
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200200 */
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100201static void set_server_check_status(struct server *s, short status, char *desc)
202{
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200203 if (status == HCHK_STATUS_START) {
204 s->result = SRV_CHK_UNKNOWN; /* no result yet */
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200205 s->check.desc[0] = '\0';
206 s->check.start = now;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200207 return;
208 }
209
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200210 if (!s->check.status)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200211 return;
212
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200213 if (desc && *desc) {
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200214 strncpy(s->check.desc, desc, HCHK_DESC_LEN-1);
215 s->check.desc[HCHK_DESC_LEN-1] = '\0';
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200216 } else
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200217 s->check.desc[0] = '\0';
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200218
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200219 s->check.status = status;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200220 if (check_statuses[status].result)
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100221 s->result = check_statuses[status].result;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200222
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100223 if (status == HCHK_STATUS_HANA)
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200224 s->check.duration = -1;
225 else if (!tv_iszero(&s->check.start)) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200226 /* set_server_check_status() may be called more than once */
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200227 s->check.duration = tv_ms_elapsed(&s->check.start, &now);
228 tv_zero(&s->check.start);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200229 }
230
231 if (s->proxy->options2 & PR_O2_LOGHCHKS &&
Willy Tarreaud3aac702012-11-23 11:32:12 +0100232 (((s->health != 0) && (s->result & SRV_CHK_FAILED)) ||
233 ((s->health != s->rise + s->fall - 1) && (s->result & SRV_CHK_PASSED)) ||
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200234 ((s->state & SRV_GOINGDOWN) && !(s->result & SRV_CHK_DISABLE)) ||
235 (!(s->state & SRV_GOINGDOWN) && (s->result & SRV_CHK_DISABLE)))) {
236
237 int health, rise, fall, state;
238
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100239 chunk_reset(&trash);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200240
241 /* FIXME begin: calculate local version of the health/rise/fall/state */
242 health = s->health;
243 rise = s->rise;
244 fall = s->fall;
245 state = s->state;
246
Willy Tarreaud3aac702012-11-23 11:32:12 +0100247 if (s->result & SRV_CHK_FAILED) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200248 if (health > rise) {
249 health--; /* still good */
250 } else {
251 if (health == rise)
252 state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
253
254 health = 0;
255 }
256 }
257
Willy Tarreaud3aac702012-11-23 11:32:12 +0100258 if (s->result & SRV_CHK_PASSED) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200259 if (health < rise + fall - 1) {
260 health++; /* was bad, stays for a while */
261
262 if (health == rise)
263 state |= SRV_RUNNING;
264
265 if (health >= rise)
266 health = rise + fall - 1; /* OK now */
267 }
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100268
269 /* clear consecutive_errors if observing is enabled */
270 if (s->onerror)
271 s->consecutive_errors = 0;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200272 }
273 /* FIXME end: calculate local version of the health/rise/fall/state */
274
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100275 chunk_appendf(&trash,
276 "Health check for %sserver %s/%s %s%s",
277 s->state & SRV_BACKUP ? "backup " : "",
278 s->proxy->id, s->id,
279 (s->result & SRV_CHK_DISABLE)?"conditionally ":"",
Willy Tarreaud3aac702012-11-23 11:32:12 +0100280 (s->result & SRV_CHK_PASSED)?"succeeded":"failed");
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200281
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100282 server_status_printf(&trash, s, SSP_O_HCHK, -1);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200283
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100284 chunk_appendf(&trash, ", status: %d/%d %s",
285 (state & SRV_RUNNING) ? (health - rise + 1) : (health),
286 (state & SRV_RUNNING) ? (fall) : (rise),
287 (state & SRV_RUNNING)?"UP":"DOWN");
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200288
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100289 Warning("%s.\n", trash.str);
290 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200291 }
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200292}
293
Willy Tarreau48494c02007-11-30 10:41:39 +0100294/* sends a log message when a backend goes down, and also sets last
295 * change date.
296 */
297static void set_backend_down(struct proxy *be)
298{
299 be->last_change = now.tv_sec;
300 be->down_trans++;
301
302 Alert("%s '%s' has no server available!\n", proxy_type_str(be), be->id);
303 send_log(be, LOG_EMERG, "%s %s has no server available!\n", proxy_type_str(be), be->id);
304}
305
306/* Redistribute pending connections when a server goes down. The number of
307 * connections redistributed is returned.
308 */
309static int redistribute_pending(struct server *s)
310{
311 struct pendconn *pc, *pc_bck, *pc_end;
312 int xferred = 0;
313
314 FOREACH_ITEM_SAFE(pc, pc_bck, &s->pendconns, pc_end, struct pendconn *, list) {
315 struct session *sess = pc->sess;
Willy Tarreau4de91492010-01-22 19:10:05 +0100316 if ((sess->be->options & (PR_O_REDISP|PR_O_PERSIST)) == PR_O_REDISP &&
317 !(sess->flags & SN_FORCE_PRST)) {
Willy Tarreau48494c02007-11-30 10:41:39 +0100318 /* The REDISP option was specified. We will ignore
319 * cookie and force to balance or use the dispatcher.
320 */
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100321
Krzysztof Piotr Oledzki5a329cf2008-02-22 03:50:19 +0100322 /* it's left to the dispatcher to choose a server */
Willy Tarreau48494c02007-11-30 10:41:39 +0100323 sess->flags &= ~(SN_DIRECT | SN_ASSIGNED | SN_ADDR_SET);
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100324
Willy Tarreau48494c02007-11-30 10:41:39 +0100325 pendconn_free(pc);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200326 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100327 xferred++;
328 }
329 }
330 return xferred;
331}
332
333/* Check for pending connections at the backend, and assign some of them to
334 * the server coming up. The server's weight is checked before being assigned
335 * connections it may not be able to handle. The total number of transferred
336 * connections is returned.
337 */
338static int check_for_pending(struct server *s)
339{
340 int xferred;
341
342 if (!s->eweight)
343 return 0;
344
345 for (xferred = 0; !s->maxconn || xferred < srv_dynamic_maxconn(s); xferred++) {
346 struct session *sess;
347 struct pendconn *p;
348
349 p = pendconn_from_px(s->proxy);
350 if (!p)
351 break;
Willy Tarreau3fdb3662012-11-12 00:42:33 +0100352 p->sess->target = &s->obj_type;
Willy Tarreau48494c02007-11-30 10:41:39 +0100353 sess = p->sess;
354 pendconn_free(p);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200355 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100356 }
357 return xferred;
358}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200359
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700360/* Shutdown all connections of a server. The caller must pass a termination
361 * code in <why>, which must be one of SN_ERR_* indicating the reason for the
362 * shutdown.
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900363 */
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700364static void shutdown_sessions(struct server *srv, int why)
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900365{
366 struct session *session, *session_bck;
367
Willy Tarreaua2a64e92011-09-07 23:01:56 +0200368 list_for_each_entry_safe(session, session_bck, &srv->actconns, by_srv)
369 if (session->srv_conn == srv)
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700370 session_shutdown(session, why);
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900371}
372
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700373/* Shutdown all connections of all backup servers of a proxy. The caller must
374 * pass a termination code in <why>, which must be one of SN_ERR_* indicating
375 * the reason for the shutdown.
376 */
377static void shutdown_backup_sessions(struct proxy *px, int why)
378{
379 struct server *srv;
380
381 for (srv = px->srv; srv != NULL; srv = srv->next)
382 if (srv->state & SRV_BACKUP)
383 shutdown_sessions(srv, why);
384}
385
Willy Tarreaubaaee002006-06-26 02:48:02 +0200386/* Sets server <s> down, notifies by all available means, recounts the
387 * remaining servers on the proxy and transfers queued sessions whenever
Willy Tarreau5af3a692007-07-24 23:32:33 +0200388 * possible to other servers. It automatically recomputes the number of
389 * servers, but not the map.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200390 */
Cyril Bontécd19e512010-01-31 22:34:03 +0100391void set_server_down(struct server *s)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200392{
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100393 struct server *srv;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200394 int xferred;
395
Cyril Bontécd19e512010-01-31 22:34:03 +0100396 if (s->state & SRV_MAINTAIN) {
397 s->health = s->rise;
398 }
399
Willy Tarreau44267702011-10-28 15:35:33 +0200400 if (s->health == s->rise || s->track) {
Willy Tarreau48494c02007-11-30 10:41:39 +0100401 int srv_was_paused = s->state & SRV_GOINGDOWN;
Willy Tarreaud64d2252010-10-17 17:16:42 +0200402 int prev_srv_count = s->proxy->srv_bck + s->proxy->srv_act;
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200403
404 s->last_change = now.tv_sec;
Willy Tarreau48494c02007-11-30 10:41:39 +0100405 s->state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
Willy Tarreau9580d162012-05-19 19:07:40 +0200406 if (s->proxy->lbprm.set_server_status_down)
407 s->proxy->lbprm.set_server_status_down(s);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200408
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900409 if (s->onmarkeddown & HANA_ONMARKEDDOWN_SHUTDOWNSESSIONS)
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700410 shutdown_sessions(s, SN_ERR_DOWN);
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900411
Willy Tarreaubaaee002006-06-26 02:48:02 +0200412 /* we might have sessions queued on this server and waiting for
413 * a connection. Those which are redispatchable will be queued
414 * to another server or to the proxy itself.
415 */
Willy Tarreau48494c02007-11-30 10:41:39 +0100416 xferred = redistribute_pending(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100417
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100418 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100419
Cyril Bontécd19e512010-01-31 22:34:03 +0100420 if (s->state & SRV_MAINTAIN) {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100421 chunk_appendf(&trash,
422 "%sServer %s/%s is DOWN for maintenance", s->state & SRV_BACKUP ? "Backup " : "",
423 s->proxy->id, s->id);
Cyril Bontécd19e512010-01-31 22:34:03 +0100424 } else {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100425 chunk_appendf(&trash,
426 "%sServer %s/%s is DOWN", s->state & SRV_BACKUP ? "Backup " : "",
427 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100428
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100429 server_status_printf(&trash, s,
430 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? SSP_O_HCHK : 0),
431 xferred);
Cyril Bontécd19e512010-01-31 22:34:03 +0100432 }
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100433 Warning("%s.\n", trash.str);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200434
Willy Tarreau48494c02007-11-30 10:41:39 +0100435 /* we don't send an alert if the server was previously paused */
436 if (srv_was_paused)
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100437 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Willy Tarreau48494c02007-11-30 10:41:39 +0100438 else
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100439 send_log(s->proxy, LOG_ALERT, "%s.\n", trash.str);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200440
Willy Tarreaud64d2252010-10-17 17:16:42 +0200441 if (prev_srv_count && s->proxy->srv_bck == 0 && s->proxy->srv_act == 0)
Willy Tarreau48494c02007-11-30 10:41:39 +0100442 set_backend_down(s->proxy);
443
Krzysztof Piotr Oledzki052d4fd2009-10-04 14:52:57 +0200444 s->counters.down_trans++;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100445
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100446 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100447 for(srv = s->tracknext; srv; srv = srv->tracknext)
Cyril Bontécd19e512010-01-31 22:34:03 +0100448 if (! (srv->state & SRV_MAINTAIN))
449 /* Only notify tracking servers that are not already in maintenance. */
450 set_server_down(srv);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200451 }
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100452
Willy Tarreaubaaee002006-06-26 02:48:02 +0200453 s->health = 0; /* failure */
454}
455
Cyril Bontécd19e512010-01-31 22:34:03 +0100456void set_server_up(struct server *s) {
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100457
458 struct server *srv;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100459 int xferred;
Willy Tarreau45446782012-03-09 17:16:09 +0100460 unsigned int old_state = s->state;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100461
Cyril Bontécd19e512010-01-31 22:34:03 +0100462 if (s->state & SRV_MAINTAIN) {
463 s->health = s->rise;
464 }
465
Willy Tarreau44267702011-10-28 15:35:33 +0200466 if (s->health == s->rise || s->track) {
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100467 if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0) {
468 if (s->proxy->last_change < now.tv_sec) // ignore negative times
469 s->proxy->down_time += now.tv_sec - s->proxy->last_change;
470 s->proxy->last_change = now.tv_sec;
471 }
472
473 if (s->last_change < now.tv_sec) // ignore negative times
474 s->down_time += now.tv_sec - s->last_change;
475
476 s->last_change = now.tv_sec;
477 s->state |= SRV_RUNNING;
Willy Tarreau45446782012-03-09 17:16:09 +0100478 s->state &= ~SRV_MAINTAIN;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100479
480 if (s->slowstart > 0) {
481 s->state |= SRV_WARMINGUP;
482 if (s->proxy->lbprm.algo & BE_LB_PROP_DYN) {
483 /* For dynamic algorithms, start at the first step of the weight,
484 * without multiplying by BE_WEIGHT_SCALE.
485 */
486 s->eweight = s->uweight;
487 if (s->proxy->lbprm.update_server_eweight)
488 s->proxy->lbprm.update_server_eweight(s);
489 }
Willy Tarreau2e993902011-10-31 11:53:20 +0100490 task_schedule(s->warmup, tick_add(now_ms, MS_TO_TICKS(MAX(1000, s->slowstart / 20))));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100491 }
Willy Tarreau9580d162012-05-19 19:07:40 +0200492 if (s->proxy->lbprm.set_server_status_up)
493 s->proxy->lbprm.set_server_status_up(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100494
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700495 /* If the server is set with "on-marked-up shutdown-backup-sessions",
496 * and it's not a backup server and its effective weight is > 0,
497 * then it can accept new connections, so we shut down all sessions
498 * on all backup servers.
499 */
500 if ((s->onmarkedup & HANA_ONMARKEDUP_SHUTDOWNBACKUPSESSIONS) &&
501 !(s->state & SRV_BACKUP) && s->eweight)
502 shutdown_backup_sessions(s->proxy, SN_ERR_UP);
503
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100504 /* check if we can handle some connections queued at the proxy. We
505 * will take as many as we can handle.
506 */
507 xferred = check_for_pending(s);
508
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100509 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100510
Willy Tarreau45446782012-03-09 17:16:09 +0100511 if (old_state & SRV_MAINTAIN) {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100512 chunk_appendf(&trash,
513 "%sServer %s/%s is UP (leaving maintenance)", s->state & SRV_BACKUP ? "Backup " : "",
514 s->proxy->id, s->id);
Cyril Bontécd19e512010-01-31 22:34:03 +0100515 } else {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100516 chunk_appendf(&trash,
517 "%sServer %s/%s is UP", s->state & SRV_BACKUP ? "Backup " : "",
518 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100519
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100520 server_status_printf(&trash, s,
521 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? SSP_O_HCHK : 0),
522 xferred);
Cyril Bontécd19e512010-01-31 22:34:03 +0100523 }
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100524
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100525 Warning("%s.\n", trash.str);
526 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100527
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100528 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100529 for(srv = s->tracknext; srv; srv = srv->tracknext)
Cyril Bontécd19e512010-01-31 22:34:03 +0100530 if (! (srv->state & SRV_MAINTAIN))
531 /* Only notify tracking servers if they're not in maintenance. */
532 set_server_up(srv);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100533 }
534
535 if (s->health >= s->rise)
536 s->health = s->rise + s->fall - 1; /* OK now */
537
538}
539
540static void set_server_disabled(struct server *s) {
541
542 struct server *srv;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100543 int xferred;
544
545 s->state |= SRV_GOINGDOWN;
Willy Tarreau9580d162012-05-19 19:07:40 +0200546 if (s->proxy->lbprm.set_server_status_down)
547 s->proxy->lbprm.set_server_status_down(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100548
549 /* we might have sessions queued on this server and waiting for
550 * a connection. Those which are redispatchable will be queued
551 * to another server or to the proxy itself.
552 */
553 xferred = redistribute_pending(s);
554
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100555 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100556
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100557 chunk_appendf(&trash,
558 "Load-balancing on %sServer %s/%s is disabled",
559 s->state & SRV_BACKUP ? "Backup " : "",
560 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100561
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100562 server_status_printf(&trash, s,
563 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? SSP_O_HCHK : 0),
564 xferred);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100565
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100566 Warning("%s.\n", trash.str);
567 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100568
569 if (!s->proxy->srv_bck && !s->proxy->srv_act)
570 set_backend_down(s->proxy);
571
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100572 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100573 for(srv = s->tracknext; srv; srv = srv->tracknext)
574 set_server_disabled(srv);
575}
576
577static void set_server_enabled(struct server *s) {
578
579 struct server *srv;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100580 int xferred;
581
582 s->state &= ~SRV_GOINGDOWN;
Willy Tarreau9580d162012-05-19 19:07:40 +0200583 if (s->proxy->lbprm.set_server_status_up)
584 s->proxy->lbprm.set_server_status_up(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100585
586 /* check if we can handle some connections queued at the proxy. We
587 * will take as many as we can handle.
588 */
589 xferred = check_for_pending(s);
590
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100591 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100592
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100593 chunk_appendf(&trash,
594 "Load-balancing on %sServer %s/%s is enabled again",
595 s->state & SRV_BACKUP ? "Backup " : "",
596 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100597
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100598 server_status_printf(&trash, s,
599 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? SSP_O_HCHK : 0),
600 xferred);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100601
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100602 Warning("%s.\n", trash.str);
603 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100604
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100605 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100606 for(srv = s->tracknext; srv; srv = srv->tracknext)
607 set_server_enabled(srv);
608}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200609
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100610void health_adjust(struct server *s, short status)
611{
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100612 int failed;
613 int expire;
614
615 /* return now if observing nor health check is not enabled */
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200616 if (!s->observe || !s->check.task)
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100617 return;
618
619 if (s->observe >= HANA_OBS_SIZE)
620 return;
621
622 if (status >= HCHK_STATUS_SIZE || !analyze_statuses[status].desc)
623 return;
624
625 switch (analyze_statuses[status].lr[s->observe - 1]) {
626 case 1:
627 failed = 1;
628 break;
629
630 case 2:
631 failed = 0;
632 break;
633
634 default:
635 return;
636 }
637
638 if (!failed) {
639 /* good: clear consecutive_errors */
640 s->consecutive_errors = 0;
641 return;
642 }
643
644 s->consecutive_errors++;
645
646 if (s->consecutive_errors < s->consecutive_errors_limit)
647 return;
648
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100649 chunk_printf(&trash, "Detected %d consecutive errors, last one was: %s",
650 s->consecutive_errors, get_analyze_status(status));
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100651
652 switch (s->onerror) {
653 case HANA_ONERR_FASTINTER:
654 /* force fastinter - nothing to do here as all modes force it */
655 break;
656
657 case HANA_ONERR_SUDDTH:
658 /* simulate a pre-fatal failed health check */
659 if (s->health > s->rise)
660 s->health = s->rise + 1;
661
662 /* no break - fall through */
663
664 case HANA_ONERR_FAILCHK:
665 /* simulate a failed health check */
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100666 set_server_check_status(s, HCHK_STATUS_HANA, trash.str);
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100667
668 if (s->health > s->rise) {
669 s->health--; /* still good */
670 s->counters.failed_checks++;
671 }
672 else
673 set_server_down(s);
674
675 break;
676
677 case HANA_ONERR_MARKDWN:
678 /* mark server down */
679 s->health = s->rise;
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100680 set_server_check_status(s, HCHK_STATUS_HANA, trash.str);
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100681 set_server_down(s);
682
683 break;
684
685 default:
686 /* write a warning? */
687 break;
688 }
689
690 s->consecutive_errors = 0;
691 s->counters.failed_hana++;
692
693 if (s->fastinter) {
694 expire = tick_add(now_ms, MS_TO_TICKS(s->fastinter));
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200695 if (s->check.task->expire > expire)
696 s->check.task->expire = expire;
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100697 }
698}
699
Willy Tarreauef781042010-01-27 11:53:01 +0100700static int httpchk_build_status_header(struct server *s, char *buffer)
701{
702 int sv_state;
703 int ratio;
704 int hlen = 0;
705 const char *srv_hlt_st[7] = { "DOWN", "DOWN %d/%d",
706 "UP %d/%d", "UP",
707 "NOLB %d/%d", "NOLB",
708 "no check" };
709
710 memcpy(buffer + hlen, "X-Haproxy-Server-State: ", 24);
711 hlen += 24;
712
713 if (!(s->state & SRV_CHECKED))
714 sv_state = 6; /* should obviously never happen */
715 else if (s->state & SRV_RUNNING) {
716 if (s->health == s->rise + s->fall - 1)
717 sv_state = 3; /* UP */
718 else
719 sv_state = 2; /* going down */
720
721 if (s->state & SRV_GOINGDOWN)
722 sv_state += 2;
723 } else {
724 if (s->health)
725 sv_state = 1; /* going up */
726 else
727 sv_state = 0; /* DOWN */
728 }
729
730 hlen += sprintf(buffer + hlen,
731 srv_hlt_st[sv_state],
732 (s->state & SRV_RUNNING) ? (s->health - s->rise + 1) : (s->health),
733 (s->state & SRV_RUNNING) ? (s->fall) : (s->rise));
734
735 hlen += sprintf(buffer + hlen, "; name=%s/%s; node=%s; weight=%d/%d; scur=%d/%d; qcur=%d",
736 s->proxy->id, s->id,
737 global.node,
738 (s->eweight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv,
739 (s->proxy->lbprm.tot_weight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv,
740 s->cur_sess, s->proxy->beconn - s->proxy->nbpend,
741 s->nbpend);
742
743 if ((s->state & SRV_WARMINGUP) &&
744 now.tv_sec < s->last_change + s->slowstart &&
745 now.tv_sec >= s->last_change) {
746 ratio = MAX(1, 100 * (now.tv_sec - s->last_change) / s->slowstart);
747 hlen += sprintf(buffer + hlen, "; throttle=%d%%", ratio);
748 }
749
750 buffer[hlen++] = '\r';
751 buffer[hlen++] = '\n';
752
753 return hlen;
754}
755
Willy Tarreaubaaee002006-06-26 02:48:02 +0200756/*
757 * This function is used only for server health-checks. It handles
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200758 * the connection acknowledgement. If the proxy requires L7 health-checks,
759 * it sends the request. In other cases, it calls set_server_check_status()
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200760 * to set s->check.status, s->check.duration and s->result.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200761 */
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200762static void event_srv_chk_w(struct connection *conn)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200763{
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200764 struct server *s = conn->owner;
765 int fd = conn->t.sock.fd;
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200766 struct task *t = s->check.task;
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200767
Willy Tarreau5a78f362012-11-23 12:47:05 +0100768 if (conn->flags & (CO_FL_SOCK_WR_SH | CO_FL_DATA_WR_SH))
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200769 conn->flags |= CO_FL_ERROR;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200770
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200771 if (unlikely(conn->flags & CO_FL_ERROR)) {
Krzysztof Piotr Oledzki6492db52010-01-02 22:03:01 +0100772 int skerr, err = errno;
773 socklen_t lskerr = sizeof(skerr);
774
775 if (!getsockopt(fd, SOL_SOCKET, SO_ERROR, &skerr, &lskerr) && skerr)
776 err = skerr;
777
778 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(err));
Willy Tarreau6996e152007-04-30 14:37:43 +0200779 goto out_error;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200780 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200781
Willy Tarreau5a78f362012-11-23 12:47:05 +0100782 if (conn->flags & (CO_FL_HANDSHAKE | CO_FL_WAIT_WR))
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200783 return;
Willy Tarreau83749182007-04-15 20:56:27 +0200784
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200785 /* here, we know that the connection is established */
Willy Tarreaud3aac702012-11-23 11:32:12 +0100786 if (!(s->result & SRV_CHK_FAILED)) {
Willy Tarreaubaaee002006-06-26 02:48:02 +0200787 /* we don't want to mark 'UP' a server on which we detected an error earlier */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200788 if (s->check.bo->o) {
Willy Tarreau392e9392012-10-05 14:54:30 +0200789 conn->xprt->snd_buf(conn, s->check.bo, MSG_DONTWAIT | MSG_NOSIGNAL);
Willy Tarreauf1503172012-09-28 19:39:36 +0200790 if (conn->flags & CO_FL_ERROR) {
791 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(errno));
792 goto out_wakeup;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200793 }
Willy Tarreau5a78f362012-11-23 12:47:05 +0100794 if (s->check.bo->o) {
795 goto out_incomplete;
796 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200797 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200798
Willy Tarreau5a78f362012-11-23 12:47:05 +0100799 /* full request sent, we allow up to <timeout.check> if nonzero for a response */
800 if (s->proxy->timeout.check) {
801 t->expire = tick_add_ifset(now_ms, s->proxy->timeout.check);
802 task_queue(t);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200803 }
Willy Tarreau5a78f362012-11-23 12:47:05 +0100804 goto out_nowake;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200805 }
Willy Tarreau83749182007-04-15 20:56:27 +0200806 out_wakeup:
Willy Tarreaufdccded2008-08-29 18:19:04 +0200807 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreau83749182007-04-15 20:56:27 +0200808 out_nowake:
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200809 __conn_data_stop_send(conn); /* nothing more to write */
Willy Tarreau5a78f362012-11-23 12:47:05 +0100810 out_incomplete:
Willy Tarreau3267d362012-08-17 23:53:56 +0200811 return;
Willy Tarreau6996e152007-04-30 14:37:43 +0200812 out_error:
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200813 conn->flags |= CO_FL_ERROR;
Willy Tarreau6996e152007-04-30 14:37:43 +0200814 goto out_wakeup;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200815}
816
817
818/*
Willy Tarreauf3c69202006-07-09 16:42:34 +0200819 * This function is used only for server health-checks. It handles the server's
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +0200820 * reply to an HTTP request, SSL HELLO or MySQL client Auth. It calls
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200821 * set_server_check_status() to update s->check.status, s->check.duration
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +0200822 * and s->result.
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200823
824 * The set_server_check_status function is called with HCHK_STATUS_L7OKD if
825 * an HTTP server replies HTTP 2xx or 3xx (valid responses), if an SMTP server
826 * returns 2xx, HCHK_STATUS_L6OK if an SSL server returns at least 5 bytes in
827 * response to an SSL HELLO (the principle is that this is enough to
828 * distinguish between an SSL server and a pure TCP relay). All other cases will
829 * call it with a proper error status like HCHK_STATUS_L7STS, HCHK_STATUS_L6RSP,
830 * etc.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200831 */
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200832static void event_srv_chk_r(struct connection *conn)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200833{
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200834 struct server *s = conn->owner;
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200835 struct task *t = s->check.task;
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200836 char *desc;
Willy Tarreau03938182010-03-17 21:52:07 +0100837 int done;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +0200838 unsigned short msglen;
Willy Tarreau83749182007-04-15 20:56:27 +0200839
Willy Tarreaud3aac702012-11-23 11:32:12 +0100840 if (unlikely((s->result & SRV_CHK_FAILED) || (conn->flags & CO_FL_ERROR))) {
Willy Tarreau83749182007-04-15 20:56:27 +0200841 /* in case of TCP only, this tells us if the connection failed */
Willy Tarreaud3aac702012-11-23 11:32:12 +0100842 if (!(s->result & SRV_CHK_FAILED))
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200843 set_server_check_status(s, HCHK_STATUS_SOCKERR, NULL);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200844
Willy Tarreau83749182007-04-15 20:56:27 +0200845 goto out_wakeup;
846 }
847
Willy Tarreau5a78f362012-11-23 12:47:05 +0100848 if (conn->flags & (CO_FL_HANDSHAKE | CO_FL_WAIT_RD))
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200849 return;
850
Willy Tarreau83749182007-04-15 20:56:27 +0200851 /* Warning! Linux returns EAGAIN on SO_ERROR if data are still available
852 * but the connection was closed on the remote end. Fortunately, recv still
853 * works correctly and we don't need to do the getsockopt() on linux.
854 */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000855
856 /* Set buffer to point to the end of the data already read, and check
857 * that there is free space remaining. If the buffer is full, proceed
858 * with running the checks without attempting another socket read.
859 */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000860
Willy Tarreau03938182010-03-17 21:52:07 +0100861 done = 0;
Nick Chalk57b1bf72010-03-16 15:50:46 +0000862
Willy Tarreau392e9392012-10-05 14:54:30 +0200863 conn->xprt->rcv_buf(conn, s->check.bi, s->check.bi->size);
Willy Tarreauf1503172012-09-28 19:39:36 +0200864 if (conn->flags & (CO_FL_ERROR | CO_FL_SOCK_RD_SH | CO_FL_DATA_RD_SH)) {
Willy Tarreau03938182010-03-17 21:52:07 +0100865 done = 1;
Willy Tarreauf1503172012-09-28 19:39:36 +0200866 if ((conn->flags & CO_FL_ERROR) && !s->check.bi->i) {
867 /* Report network errors only if we got no other data. Otherwise
868 * we'll let the upper layers decide whether the response is OK
869 * or not. It is very common that an RST sent by the server is
870 * reported as an error just after the last data chunk.
871 */
Willy Tarreaud3aac702012-11-23 11:32:12 +0100872 if (!(s->result & SRV_CHK_FAILED))
Willy Tarreauc1a07962010-03-16 20:55:43 +0100873 set_server_check_status(s, HCHK_STATUS_SOCKERR, NULL);
874 goto out_wakeup;
875 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200876 }
877
Willy Tarreau03938182010-03-17 21:52:07 +0100878 /* Intermediate or complete response received.
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200879 * Terminate string in check.bi->data buffer.
Willy Tarreau03938182010-03-17 21:52:07 +0100880 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200881 if (s->check.bi->i < s->check.bi->size)
882 s->check.bi->data[s->check.bi->i] = '\0';
Willy Tarreau03938182010-03-17 21:52:07 +0100883 else {
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200884 s->check.bi->data[s->check.bi->i - 1] = '\0';
Willy Tarreau03938182010-03-17 21:52:07 +0100885 done = 1; /* buffer full, don't wait for more data */
886 }
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200887
Nick Chalk57b1bf72010-03-16 15:50:46 +0000888 /* Run the checks... */
Willy Tarreau1620ec32011-08-06 17:05:02 +0200889 switch (s->proxy->options2 & PR_O2_CHK_ANY) {
890 case PR_O2_HTTP_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200891 if (!done && s->check.bi->i < strlen("HTTP/1.0 000\r"))
Willy Tarreau03938182010-03-17 21:52:07 +0100892 goto wait_more_data;
893
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100894 /* Check if the server speaks HTTP 1.X */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200895 if ((s->check.bi->i < strlen("HTTP/1.0 000\r")) ||
896 (memcmp(s->check.bi->data, "HTTP/1.", 7) != 0 ||
897 (*(s->check.bi->data + 12) != ' ' && *(s->check.bi->data + 12) != '\r')) ||
898 !isdigit((unsigned char) *(s->check.bi->data + 9)) || !isdigit((unsigned char) *(s->check.bi->data + 10)) ||
899 !isdigit((unsigned char) *(s->check.bi->data + 11))) {
900 cut_crlf(s->check.bi->data);
901 set_server_check_status(s, HCHK_STATUS_L7RSP, s->check.bi->data);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200902
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100903 goto out_wakeup;
904 }
905
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200906 s->check.code = str2uic(s->check.bi->data + 9);
907 desc = ltrim(s->check.bi->data + 12, ' ');
Nick Chalk57b1bf72010-03-16 15:50:46 +0000908
Willy Tarreaubd741542010-03-16 18:46:54 +0100909 if ((s->proxy->options & PR_O_DISABLE404) &&
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200910 (s->state & SRV_RUNNING) && (s->check.code == 404)) {
Nick Chalk57b1bf72010-03-16 15:50:46 +0000911 /* 404 may be accepted as "stopping" only if the server was up */
912 cut_crlf(desc);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200913 set_server_check_status(s, HCHK_STATUS_L7OKCD, desc);
Nick Chalk57b1bf72010-03-16 15:50:46 +0000914 }
Willy Tarreaubd741542010-03-16 18:46:54 +0100915 else if (s->proxy->options2 & PR_O2_EXP_TYPE) {
916 /* Run content verification check... We know we have at least 13 chars */
917 if (!httpchk_expect(s, done))
918 goto wait_more_data;
919 }
920 /* check the reply : HTTP/1.X 2xx and 3xx are OK */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200921 else if (*(s->check.bi->data + 9) == '2' || *(s->check.bi->data + 9) == '3') {
Willy Tarreaubd741542010-03-16 18:46:54 +0100922 cut_crlf(desc);
923 set_server_check_status(s, HCHK_STATUS_L7OKD, desc);
924 }
Nick Chalk57b1bf72010-03-16 15:50:46 +0000925 else {
926 cut_crlf(desc);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200927 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
Nick Chalk57b1bf72010-03-16 15:50:46 +0000928 }
Willy Tarreau1620ec32011-08-06 17:05:02 +0200929 break;
930
931 case PR_O2_SSL3_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200932 if (!done && s->check.bi->i < 5)
Willy Tarreau03938182010-03-17 21:52:07 +0100933 goto wait_more_data;
934
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100935 /* Check for SSLv3 alert or handshake */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200936 if ((s->check.bi->i >= 5) && (*s->check.bi->data == 0x15 || *s->check.bi->data == 0x16))
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200937 set_server_check_status(s, HCHK_STATUS_L6OK, NULL);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200938 else
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200939 set_server_check_status(s, HCHK_STATUS_L6RSP, NULL);
Willy Tarreau1620ec32011-08-06 17:05:02 +0200940 break;
941
942 case PR_O2_SMTP_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200943 if (!done && s->check.bi->i < strlen("000\r"))
Willy Tarreau03938182010-03-17 21:52:07 +0100944 goto wait_more_data;
945
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200946 /* Check if the server speaks SMTP */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200947 if ((s->check.bi->i < strlen("000\r")) ||
948 (*(s->check.bi->data + 3) != ' ' && *(s->check.bi->data + 3) != '\r') ||
949 !isdigit((unsigned char) *s->check.bi->data) || !isdigit((unsigned char) *(s->check.bi->data + 1)) ||
950 !isdigit((unsigned char) *(s->check.bi->data + 2))) {
951 cut_crlf(s->check.bi->data);
952 set_server_check_status(s, HCHK_STATUS_L7RSP, s->check.bi->data);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200953
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200954 goto out_wakeup;
955 }
956
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200957 s->check.code = str2uic(s->check.bi->data);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200958
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200959 desc = ltrim(s->check.bi->data + 3, ' ');
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200960 cut_crlf(desc);
961
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100962 /* Check for SMTP code 2xx (should be 250) */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200963 if (*s->check.bi->data == '2')
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200964 set_server_check_status(s, HCHK_STATUS_L7OKD, desc);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200965 else
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200966 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
Willy Tarreau1620ec32011-08-06 17:05:02 +0200967 break;
968
969 case PR_O2_PGSQL_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200970 if (!done && s->check.bi->i < 9)
Rauf Kuliyev38b41562011-01-04 15:14:13 +0100971 goto wait_more_data;
972
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200973 if (s->check.bi->data[0] == 'R') {
Rauf Kuliyev38b41562011-01-04 15:14:13 +0100974 set_server_check_status(s, HCHK_STATUS_L7OKD, "PostgreSQL server is ok");
975 }
976 else {
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200977 if ((s->check.bi->data[0] == 'E') && (s->check.bi->data[5]!=0) && (s->check.bi->data[6]!=0))
978 desc = &s->check.bi->data[6];
Rauf Kuliyev38b41562011-01-04 15:14:13 +0100979 else
980 desc = "PostgreSQL unknown error";
981
982 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
983 }
Willy Tarreau1620ec32011-08-06 17:05:02 +0200984 break;
985
986 case PR_O2_REDIS_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200987 if (!done && s->check.bi->i < 7)
Hervé COMMOWICKec032d62011-08-05 16:23:48 +0200988 goto wait_more_data;
989
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200990 if (strcmp(s->check.bi->data, "+PONG\r\n") == 0) {
Hervé COMMOWICKec032d62011-08-05 16:23:48 +0200991 set_server_check_status(s, HCHK_STATUS_L7OKD, "Redis server is ok");
992 }
993 else {
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200994 set_server_check_status(s, HCHK_STATUS_L7STS, s->check.bi->data);
Hervé COMMOWICKec032d62011-08-05 16:23:48 +0200995 }
Willy Tarreau1620ec32011-08-06 17:05:02 +0200996 break;
997
998 case PR_O2_MYSQL_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +0200999 if (!done && s->check.bi->i < 5)
Willy Tarreau03938182010-03-17 21:52:07 +01001000 goto wait_more_data;
1001
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001002 if (s->proxy->check_len == 0) { // old mode
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001003 if (*(s->check.bi->data + 4) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001004 /* We set the MySQL Version in description for information purpose
1005 * FIXME : it can be cool to use MySQL Version for other purpose,
1006 * like mark as down old MySQL server.
1007 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001008 if (s->check.bi->i > 51) {
1009 desc = ltrim(s->check.bi->data + 5, ' ');
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001010 set_server_check_status(s, HCHK_STATUS_L7OKD, desc);
1011 }
1012 else {
1013 if (!done)
1014 goto wait_more_data;
1015 /* it seems we have a OK packet but without a valid length,
1016 * it must be a protocol error
1017 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001018 set_server_check_status(s, HCHK_STATUS_L7RSP, s->check.bi->data);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001019 }
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001020 }
1021 else {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001022 /* An error message is attached in the Error packet */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001023 desc = ltrim(s->check.bi->data + 7, ' ');
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001024 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
1025 }
1026 } else {
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001027 unsigned int first_packet_len = ((unsigned int) *s->check.bi->data) +
1028 (((unsigned int) *(s->check.bi->data + 1)) << 8) +
1029 (((unsigned int) *(s->check.bi->data + 2)) << 16);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001030
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001031 if (s->check.bi->i == first_packet_len + 4) {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001032 /* MySQL Error packet always begin with field_count = 0xff */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001033 if (*(s->check.bi->data + 4) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001034 /* We have only one MySQL packet and it is a Handshake Initialization packet
1035 * but we need to have a second packet to know if it is alright
1036 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001037 if (!done && s->check.bi->i < first_packet_len + 5)
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001038 goto wait_more_data;
1039 }
1040 else {
1041 /* We have only one packet and it is an Error packet,
1042 * an error message is attached, so we can display it
1043 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001044 desc = &s->check.bi->data[7];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001045 //Warning("onlyoneERR: %s\n", desc);
1046 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
1047 }
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001048 } else if (s->check.bi->i > first_packet_len + 4) {
1049 unsigned int second_packet_len = ((unsigned int) *(s->check.bi->data + first_packet_len + 4)) +
1050 (((unsigned int) *(s->check.bi->data + first_packet_len + 5)) << 8) +
1051 (((unsigned int) *(s->check.bi->data + first_packet_len + 6)) << 16);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001052
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001053 if (s->check.bi->i == first_packet_len + 4 + second_packet_len + 4 ) {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001054 /* We have 2 packets and that's good */
1055 /* Check if the second packet is a MySQL Error packet or not */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001056 if (*(s->check.bi->data + first_packet_len + 8) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001057 /* No error packet */
1058 /* We set the MySQL Version in description for information purpose */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001059 desc = &s->check.bi->data[5];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001060 //Warning("2packetOK: %s\n", desc);
1061 set_server_check_status(s, HCHK_STATUS_L7OKD, desc);
1062 }
1063 else {
1064 /* An error message is attached in the Error packet
1065 * so we can display it ! :)
1066 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001067 desc = &s->check.bi->data[first_packet_len+11];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001068 //Warning("2packetERR: %s\n", desc);
1069 set_server_check_status(s, HCHK_STATUS_L7STS, desc);
1070 }
1071 }
1072 }
1073 else {
Willy Tarreau03938182010-03-17 21:52:07 +01001074 if (!done)
1075 goto wait_more_data;
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001076 /* it seems we have a Handshake Initialization packet but without a valid length,
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001077 * it must be a protocol error
1078 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001079 desc = &s->check.bi->data[5];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001080 //Warning("protoerr: %s\n", desc);
1081 set_server_check_status(s, HCHK_STATUS_L7RSP, desc);
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001082 }
1083 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001084 break;
1085
1086 case PR_O2_LDAP_CHK:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001087 if (!done && s->check.bi->i < 14)
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001088 goto wait_more_data;
1089
1090 /* Check if the server speaks LDAP (ASN.1/BER)
1091 * http://en.wikipedia.org/wiki/Basic_Encoding_Rules
1092 * http://tools.ietf.org/html/rfc4511
1093 */
1094
1095 /* http://tools.ietf.org/html/rfc4511#section-4.1.1
1096 * LDAPMessage: 0x30: SEQUENCE
1097 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001098 if ((s->check.bi->i < 14) || (*(s->check.bi->data) != '\x30')) {
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001099 set_server_check_status(s, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
1100 }
1101 else {
1102 /* size of LDAPMessage */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001103 msglen = (*(s->check.bi->data + 1) & 0x80) ? (*(s->check.bi->data + 1) & 0x7f) : 0;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001104
1105 /* http://tools.ietf.org/html/rfc4511#section-4.2.2
1106 * messageID: 0x02 0x01 0x01: INTEGER 1
1107 * protocolOp: 0x61: bindResponse
1108 */
1109 if ((msglen > 2) ||
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001110 (memcmp(s->check.bi->data + 2 + msglen, "\x02\x01\x01\x61", 4) != 0)) {
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001111 set_server_check_status(s, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
1112
1113 goto out_wakeup;
1114 }
1115
1116 /* size of bindResponse */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001117 msglen += (*(s->check.bi->data + msglen + 6) & 0x80) ? (*(s->check.bi->data + msglen + 6) & 0x7f) : 0;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001118
1119 /* http://tools.ietf.org/html/rfc4511#section-4.1.9
1120 * ldapResult: 0x0a 0x01: ENUMERATION
1121 */
1122 if ((msglen > 4) ||
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001123 (memcmp(s->check.bi->data + 7 + msglen, "\x0a\x01", 2) != 0)) {
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001124 set_server_check_status(s, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
1125
1126 goto out_wakeup;
1127 }
1128
1129 /* http://tools.ietf.org/html/rfc4511#section-4.1.9
1130 * resultCode
1131 */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001132 s->check.code = *(s->check.bi->data + msglen + 9);
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001133 if (s->check.code) {
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001134 set_server_check_status(s, HCHK_STATUS_L7STS, "See RFC: http://tools.ietf.org/html/rfc4511#section-4.1.9");
1135 } else {
1136 set_server_check_status(s, HCHK_STATUS_L7OKD, "Success");
1137 }
1138 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001139 break;
1140
1141 default:
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001142 /* other checks are valid if the connection succeeded anyway */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001143 set_server_check_status(s, HCHK_STATUS_L4OK, NULL);
Willy Tarreau1620ec32011-08-06 17:05:02 +02001144 break;
1145 } /* switch */
Willy Tarreau83749182007-04-15 20:56:27 +02001146
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001147 out_wakeup:
Willy Tarreaud3aac702012-11-23 11:32:12 +01001148 if (s->result & SRV_CHK_FAILED)
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001149 conn->flags |= CO_FL_ERROR;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001150
Nick Chalk57b1bf72010-03-16 15:50:46 +00001151 /* Reset the check buffer... */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001152 *s->check.bi->data = '\0';
1153 s->check.bi->i = 0;
Nick Chalk57b1bf72010-03-16 15:50:46 +00001154
Willy Tarreaufd29cc52012-11-23 09:18:20 +01001155 /* Close the connection... We absolutely want to perform a hard close
1156 * and reset the connection if some data are pending, otherwise we end
1157 * up with many TIME_WAITs and eat all the source port range quickly.
1158 * To avoid sending RSTs all the time, we first try to drain pending
1159 * data.
1160 */
Willy Tarreauf1503172012-09-28 19:39:36 +02001161 if (conn->xprt && conn->xprt->shutw)
1162 conn->xprt->shutw(conn, 0);
Willy Tarreaucfd97c62012-11-23 17:35:59 +01001163 if (conn->ctrl) {
1164 if (!(conn->flags & CO_FL_WAIT_RD))
1165 recv(conn->t.sock.fd, trash.str, trash.size, MSG_NOSIGNAL|MSG_DONTWAIT);
1166 setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
1167 (struct linger *) &nolinger, sizeof(struct linger));
1168 }
Willy Tarreaua522f802012-11-23 08:56:35 +01001169 __conn_data_stop_both(conn);
Willy Tarreaufdccded2008-08-29 18:19:04 +02001170 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreau3267d362012-08-17 23:53:56 +02001171 return;
Willy Tarreau03938182010-03-17 21:52:07 +01001172
1173 wait_more_data:
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001174 __conn_data_poll_recv(conn);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001175}
1176
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001177/*
1178 * This function is used only for server health-checks. It handles connection
1179 * status updates including errors. If necessary, it wakes the check task up.
1180 * It always returns 0.
1181 */
1182static int wake_srv_chk(struct connection *conn)
Willy Tarreau20bea422012-07-06 12:00:49 +02001183{
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001184 struct server *s = conn->owner;
Willy Tarreau20bea422012-07-06 12:00:49 +02001185
Willy Tarreau6c560da2012-11-24 11:14:45 +01001186 if (unlikely(conn->flags & CO_FL_ERROR)) {
1187 /* Note that we might as well have been woken up by a handshake handler */
1188 s->result |= SRV_CHK_FAILED;
1189 __conn_data_stop_both(conn);
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001190 task_wakeup(s->check.task, TASK_WOKEN_IO);
Willy Tarreau6c560da2012-11-24 11:14:45 +01001191 }
Willy Tarreau20bea422012-07-06 12:00:49 +02001192
Willy Tarreau2b199c92012-11-23 17:32:21 +01001193 if (s->result & (SRV_CHK_FAILED|SRV_CHK_PASSED))
1194 conn_full_close(conn);
Willy Tarreau3267d362012-08-17 23:53:56 +02001195 return 0;
Willy Tarreau20bea422012-07-06 12:00:49 +02001196}
1197
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001198struct data_cb check_conn_cb = {
1199 .recv = event_srv_chk_r,
1200 .send = event_srv_chk_w,
1201 .wake = wake_srv_chk,
1202};
1203
Willy Tarreaubaaee002006-06-26 02:48:02 +02001204/*
Willy Tarreau2e993902011-10-31 11:53:20 +01001205 * updates the server's weight during a warmup stage. Once the final weight is
1206 * reached, the task automatically stops. Note that any server status change
1207 * must have updated s->last_change accordingly.
1208 */
1209static struct task *server_warmup(struct task *t)
1210{
1211 struct server *s = t->context;
1212
1213 /* by default, plan on stopping the task */
1214 t->expire = TICK_ETERNITY;
1215 if ((s->state & (SRV_RUNNING|SRV_WARMINGUP|SRV_MAINTAIN)) != (SRV_RUNNING|SRV_WARMINGUP))
1216 return t;
1217
1218 if (now.tv_sec < s->last_change || now.tv_sec >= s->last_change + s->slowstart) {
1219 /* go to full throttle if the slowstart interval is reached */
1220 s->state &= ~SRV_WARMINGUP;
1221 if (s->proxy->lbprm.algo & BE_LB_PROP_DYN)
1222 s->eweight = s->uweight * BE_WEIGHT_SCALE;
1223 if (s->proxy->lbprm.update_server_eweight)
1224 s->proxy->lbprm.update_server_eweight(s);
1225 }
1226 else if (s->proxy->lbprm.algo & BE_LB_PROP_DYN) {
1227 /* for dynamic algorithms, let's slowly update the weight */
1228 s->eweight = (BE_WEIGHT_SCALE * (now.tv_sec - s->last_change) +
1229 s->slowstart - 1) / s->slowstart;
1230 s->eweight *= s->uweight;
1231 if (s->proxy->lbprm.update_server_eweight)
1232 s->proxy->lbprm.update_server_eweight(s);
1233 }
1234 /* Note that static algorithms are already running at full throttle */
1235
1236 /* probably that we can refill this server with a bit more connections */
1237 check_for_pending(s);
1238
1239 /* get back there in 1 second or 1/20th of the slowstart interval,
1240 * whichever is greater, resulting in small 5% steps.
1241 */
1242 if (s->state & SRV_WARMINGUP)
1243 t->expire = tick_add(now_ms, MS_TO_TICKS(MAX(1000, s->slowstart / 20)));
1244 return t;
1245}
1246
1247/*
Willy Tarreaubaaee002006-06-26 02:48:02 +02001248 * manages a server health-check. Returns
1249 * the time the task accepts to wait, or TIME_ETERNITY for infinity.
1250 */
Simon Horman63a4a822012-03-19 07:24:41 +09001251static struct task *process_chk(struct task *t)
Willy Tarreaubaaee002006-06-26 02:48:02 +02001252{
1253 struct server *s = t->context;
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001254 struct connection *conn = s->check.conn;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001255 int rv;
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001256 int ret;
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001257 int expired = tick_is_expired(t->expire, now_ms);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001258
Willy Tarreau5a78f362012-11-23 12:47:05 +01001259 if (!(s->state & SRV_CHK_RUNNING)) {
1260 /* no check currently running */
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001261 if (!expired) /* woke up too early */
Willy Tarreau26c25062009-03-08 09:38:41 +01001262 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001263
1264 /* we don't send any health-checks when the proxy is stopped or when
1265 * the server should not be checked.
1266 */
Willy Tarreau5a78f362012-11-23 12:47:05 +01001267 if (!(s->state & SRV_CHECKED) || s->proxy->state == PR_STSTOPPED || (s->state & SRV_MAINTAIN))
1268 goto reschedule;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001269
1270 /* we'll initiate a new check */
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001271 set_server_check_status(s, HCHK_STATUS_START, NULL);
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001272
Willy Tarreau5a78f362012-11-23 12:47:05 +01001273 s->state |= SRV_CHK_RUNNING;
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001274 s->check.bi->p = s->check.bi->data;
1275 s->check.bi->i = 0;
1276 s->check.bo->p = s->check.bo->data;
1277 s->check.bo->o = 0;
1278
1279 /* prepare the check buffer */
1280 if (s->proxy->options2 & PR_O2_CHK_ANY) {
1281 bo_putblk(s->check.bo, s->proxy->check_req, s->proxy->check_len);
1282
1283 /* we want to check if this host replies to HTTP or SSLv3 requests
1284 * so we'll send the request, and won't wake the checker up now.
1285 */
1286 if ((s->proxy->options2 & PR_O2_CHK_ANY) == PR_O2_SSL3_CHK) {
1287 /* SSL requires that we put Unix time in the request */
1288 int gmt_time = htonl(date.tv_sec);
1289 memcpy(s->check.bo->data + 11, &gmt_time, 4);
1290 }
1291 else if ((s->proxy->options2 & PR_O2_CHK_ANY) == PR_O2_HTTP_CHK) {
1292 if (s->proxy->options2 & PR_O2_CHK_SNDST)
Willy Tarreau19d14ef2012-10-29 16:51:55 +01001293 bo_putblk(s->check.bo, trash.str, httpchk_build_status_header(s, trash.str));
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001294 bo_putstr(s->check.bo, "\r\n");
1295 *s->check.bo->p = '\0'; /* to make gdb output easier to read */
1296 }
1297 }
1298
1299 /* prepare a new connection */
Willy Tarreaub63b5962012-11-23 16:22:08 +01001300 conn->flags = CO_FL_NONE;
Willy Tarreau14cba4b2012-11-30 17:33:05 +01001301 conn->err_code = CO_ER_NONE;
Willy Tarreau3fdb3662012-11-12 00:42:33 +01001302 conn->target = &s->obj_type;
Willy Tarreauf4288ee2012-09-28 18:13:10 +02001303 conn_prepare(conn, &check_conn_cb, s->check.proto, s->check.xprt, s);
Willy Tarreauc6f4ce82009-06-10 11:09:37 +02001304
Willy Tarreau5f2877a2012-10-26 19:57:58 +02001305 /* no client address */
1306 clear_addr(&conn->addr.from);
1307
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001308 if (is_addr(&s->check.addr))
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001309 /* we'll connect to the check addr specified on the server */
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001310 conn->addr.to = s->check.addr;
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001311 else
1312 /* we'll connect to the addr on the server */
1313 conn->addr.to = s->addr;
Willy Tarreauc6f4ce82009-06-10 11:09:37 +02001314
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001315 set_host_port(&conn->addr.to, s->check.port);
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001316
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001317 /* It can return one of :
1318 * - SN_ERR_NONE if everything's OK
1319 * - SN_ERR_SRVTO if there are no more servers
1320 * - SN_ERR_SRVCL if the connection was refused by the server
1321 * - SN_ERR_PRXCOND if the connection has been limited by the proxy (maxconn)
1322 * - SN_ERR_RESOURCE if a system resource is lacking (eg: fd limits, ports, ...)
1323 * - SN_ERR_INTERNAL for any other purely internal errors
1324 * Additionnally, in the case of SN_ERR_RESOURCE, an emergency log will be emitted.
Willy Tarreau24db47e2012-11-23 14:16:39 +01001325 * Note that we try to prevent the network stack from sending the ACK during the
Willy Tarreauf0837b22012-11-24 10:24:27 +01001326 * connect() when a pure TCP check is used (without PROXY protocol).
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001327 */
Willy Tarreauf0837b22012-11-24 10:24:27 +01001328 ret = s->check.proto->connect(conn, s->proxy->options2 & PR_O2_CHK_ANY,
1329 s->check.send_proxy ? 1 : (s->proxy->options2 & PR_O2_CHK_ANY) ? 0 : 2);
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001330 conn->flags |= CO_FL_WAKE_DATA;
Willy Tarreau6c16adc2012-10-05 00:04:16 +02001331 if (s->check.send_proxy)
1332 conn->flags |= CO_FL_LOCAL_SPROXY;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001333
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001334 switch (ret) {
1335 case SN_ERR_NONE:
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001336 /* we allow up to min(inter, timeout.connect) for a connection
1337 * to establish but only when timeout.check is set
1338 * as it may be to short for a full check otherwise
1339 */
1340 t->expire = tick_add(now_ms, MS_TO_TICKS(s->inter));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001341
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001342 if (s->proxy->timeout.check && s->proxy->timeout.connect) {
1343 int t_con = tick_add(now_ms, s->proxy->timeout.connect);
1344 t->expire = tick_first(t->expire, t_con);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001345 }
Willy Tarreaudb3b4a22012-11-23 16:32:33 +01001346 conn_data_poll_recv(conn); /* prepare for reading a possible reply */
Willy Tarreau5a78f362012-11-23 12:47:05 +01001347 goto reschedule;
1348
1349 case SN_ERR_SRVTO: /* ETIMEDOUT */
1350 case SN_ERR_SRVCL: /* ECONNREFUSED, ENETUNREACH, ... */
1351 set_server_check_status(s, HCHK_STATUS_L4CON, strerror(errno));
1352 break;
1353 case SN_ERR_PRXCOND:
1354 case SN_ERR_RESOURCE:
1355 case SN_ERR_INTERNAL:
1356 set_server_check_status(s, HCHK_STATUS_SOCKERR, NULL);
1357 break;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001358 }
1359
Willy Tarreau5a78f362012-11-23 12:47:05 +01001360 /* here, we have seen a synchronous error, no fd was allocated */
Willy Tarreau6b0a8502012-11-23 08:51:32 +01001361
Willy Tarreau5a78f362012-11-23 12:47:05 +01001362 s->state &= ~SRV_CHK_RUNNING;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001363 if (s->health > s->rise) {
1364 s->health--; /* still good */
Krzysztof Piotr Oledzki052d4fd2009-10-04 14:52:57 +02001365 s->counters.failed_checks++;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001366 }
1367 else
1368 set_server_down(s);
1369
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001370 /* we allow up to min(inter, timeout.connect) for a connection
1371 * to establish but only when timeout.check is set
1372 * as it may be to short for a full check otherwise
1373 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001374 while (tick_is_expired(t->expire, now_ms)) {
1375 int t_con;
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001376
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001377 t_con = tick_add(t->expire, s->proxy->timeout.connect);
1378 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001379
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001380 if (s->proxy->timeout.check)
1381 t->expire = tick_first(t->expire, t_con);
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001382 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001383 }
1384 else {
Willy Tarreauf1503172012-09-28 19:39:36 +02001385 /* there was a test running.
1386 * First, let's check whether there was an uncaught error,
1387 * which can happen on connect timeout or error.
1388 */
1389 if (s->result == SRV_CHK_UNKNOWN) {
Willy Tarreaub63b5962012-11-23 16:22:08 +01001390 if (expired && conn->xprt) {
1391 /* the check expired and the connection was not
1392 * yet closed, start by doing this.
1393 */
Willy Tarreaucfd97c62012-11-23 17:35:59 +01001394 if (conn->ctrl)
1395 setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
1396 (struct linger *) &nolinger, sizeof(struct linger));
Willy Tarreau2b199c92012-11-23 17:32:21 +01001397 conn_full_close(conn);
Willy Tarreaub63b5962012-11-23 16:22:08 +01001398 }
1399
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001400 if ((conn->flags & (CO_FL_CONNECTED|CO_FL_WAIT_L4_CONN)) == CO_FL_WAIT_L4_CONN) {
1401 /* L4 not established (yet) */
Willy Tarreauf1503172012-09-28 19:39:36 +02001402 if (conn->flags & CO_FL_ERROR)
1403 set_server_check_status(s, HCHK_STATUS_L4CON, NULL);
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001404 else if (expired)
Willy Tarreauf1503172012-09-28 19:39:36 +02001405 set_server_check_status(s, HCHK_STATUS_L4TOUT, NULL);
1406 }
Willy Tarreau5a78f362012-11-23 12:47:05 +01001407 else if ((conn->flags & (CO_FL_CONNECTED|CO_FL_WAIT_L6_CONN)) == CO_FL_WAIT_L6_CONN) {
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001408 /* L6 not established (yet) */
Willy Tarreauf1503172012-09-28 19:39:36 +02001409 if (conn->flags & CO_FL_ERROR)
1410 set_server_check_status(s, HCHK_STATUS_L6RSP, NULL);
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001411 else if (expired)
1412 set_server_check_status(s, HCHK_STATUS_L6TOUT, NULL);
1413 }
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001414 else if (!(s->proxy->options2 & PR_O2_CHK_ANY)) {
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001415 /* good connection is enough for pure TCP check */
1416 if (s->check.use_ssl)
1417 set_server_check_status(s, HCHK_STATUS_L6OK, NULL);
Willy Tarreauf1503172012-09-28 19:39:36 +02001418 else
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001419 set_server_check_status(s, HCHK_STATUS_L4OK, NULL);
1420 }
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001421 else if (expired) {
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001422 /* connection established but expired check */
1423 if ((s->proxy->options2 & PR_O2_CHK_ANY) == PR_O2_SSL3_CHK)
Willy Tarreauf1503172012-09-28 19:39:36 +02001424 set_server_check_status(s, HCHK_STATUS_L6TOUT, NULL);
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001425 else /* HTTP, SMTP, ... */
1426 set_server_check_status(s, HCHK_STATUS_L7TOUT, NULL);
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001427
Willy Tarreauf1503172012-09-28 19:39:36 +02001428 }
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001429 else
1430 goto out_wait; /* timeout not reached, wait again */
Willy Tarreauf1503172012-09-28 19:39:36 +02001431 }
1432
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001433 /* check complete or aborted */
1434
1435 if (s->result & SRV_CHK_FAILED) { /* a failure or timeout detected */
1436 if (s->health > s->rise) {
1437 s->health--; /* still good */
1438 s->counters.failed_checks++;
1439 }
1440 else
1441 set_server_down(s);
1442 }
1443 else { /* check was OK */
Willy Tarreau48494c02007-11-30 10:41:39 +01001444 /* we may have to add/remove this server from the LB group */
1445 if ((s->state & SRV_RUNNING) && (s->proxy->options & PR_O_DISABLE404)) {
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001446 if ((s->state & SRV_GOINGDOWN) && !(s->result & SRV_CHK_DISABLE))
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +01001447 set_server_enabled(s);
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001448 else if (!(s->state & SRV_GOINGDOWN) && (s->result & SRV_CHK_DISABLE))
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +01001449 set_server_disabled(s);
Willy Tarreau48494c02007-11-30 10:41:39 +01001450 }
1451
Krzysztof Oledzki85130942007-10-22 16:21:10 +02001452 if (s->health < s->rise + s->fall - 1) {
1453 s->health++; /* was bad, stays for a while */
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +01001454 set_server_up(s);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001455 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001456 }
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001457 s->state &= ~SRV_CHK_RUNNING;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001458
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001459 rv = 0;
1460 if (global.spread_checks > 0) {
1461 rv = srv_getinter(s) * global.spread_checks / 100;
1462 rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0)));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001463 }
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001464 t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(s) + rv));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001465 }
Willy Tarreau5a78f362012-11-23 12:47:05 +01001466
1467 reschedule:
1468 while (tick_is_expired(t->expire, now_ms))
1469 t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001470 out_wait:
Willy Tarreau26c25062009-03-08 09:38:41 +01001471 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001472}
1473
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001474/*
1475 * Start health-check.
1476 * Returns 0 if OK, -1 if error, and prints the error in this case.
1477 */
1478int start_checks() {
1479
1480 struct proxy *px;
1481 struct server *s;
1482 struct task *t;
1483 int nbchk=0, mininter=0, srvpos=0;
1484
Willy Tarreau2c43a1e2007-10-14 23:05:39 +02001485 /* 1- count the checkers to run simultaneously.
1486 * We also determine the minimum interval among all of those which
1487 * have an interval larger than SRV_CHK_INTER_THRES. This interval
1488 * will be used to spread their start-up date. Those which have
Jamie Gloudon801a0a32012-08-25 00:18:33 -04001489 * a shorter interval will start independently and will not dictate
Willy Tarreau2c43a1e2007-10-14 23:05:39 +02001490 * too short an interval for all others.
1491 */
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001492 for (px = proxy; px; px = px->next) {
1493 for (s = px->srv; s; s = s->next) {
1494 if (!(s->state & SRV_CHECKED))
1495 continue;
1496
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001497 if ((srv_getinter(s) >= SRV_CHK_INTER_THRES) &&
1498 (!mininter || mininter > srv_getinter(s)))
1499 mininter = srv_getinter(s);
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001500
1501 nbchk++;
1502 }
1503 }
1504
1505 if (!nbchk)
1506 return 0;
1507
1508 srand((unsigned)time(NULL));
1509
1510 /*
1511 * 2- start them as far as possible from each others. For this, we will
1512 * start them after their interval set to the min interval divided by
1513 * the number of servers, weighted by the server's position in the list.
1514 */
1515 for (px = proxy; px; px = px->next) {
1516 for (s = px->srv; s; s = s->next) {
Willy Tarreau2e993902011-10-31 11:53:20 +01001517 if (s->slowstart) {
1518 if ((t = task_new()) == NULL) {
1519 Alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
1520 return -1;
1521 }
1522 /* We need a warmup task that will be called when the server
1523 * state switches from down to up.
1524 */
1525 s->warmup = t;
1526 t->process = server_warmup;
1527 t->context = s;
1528 t->expire = TICK_ETERNITY;
1529 }
1530
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001531 if (!(s->state & SRV_CHECKED))
1532 continue;
1533
Willy Tarreau2e993902011-10-31 11:53:20 +01001534 /* one task for the checks */
Willy Tarreaua4613182009-03-21 18:13:21 +01001535 if ((t = task_new()) == NULL) {
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001536 Alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
1537 return -1;
1538 }
1539
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001540 s->check.task = t;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001541 t->process = process_chk;
1542 t->context = s;
1543
1544 /* check this every ms */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001545 t->expire = tick_add(now_ms,
1546 MS_TO_TICKS(((mininter && mininter >= srv_getinter(s)) ?
1547 mininter : srv_getinter(s)) * srvpos / nbchk));
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001548 s->check.start = now;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001549 task_queue(t);
1550
1551 srvpos++;
1552 }
1553 }
1554 return 0;
1555}
Willy Tarreaubaaee002006-06-26 02:48:02 +02001556
1557/*
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001558 * Perform content verification check on data in s->check.buffer buffer.
Willy Tarreaubd741542010-03-16 18:46:54 +01001559 * The buffer MUST be terminated by a null byte before calling this function.
1560 * Sets server status appropriately. The caller is responsible for ensuring
1561 * that the buffer contains at least 13 characters. If <done> is zero, we may
1562 * return 0 to indicate that data is required to decide of a match.
1563 */
1564static int httpchk_expect(struct server *s, int done)
1565{
1566 static char status_msg[] = "HTTP status check returned code <000>";
1567 char status_code[] = "000";
1568 char *contentptr;
1569 int crlf;
1570 int ret;
1571
1572 switch (s->proxy->options2 & PR_O2_EXP_TYPE) {
1573 case PR_O2_EXP_STS:
1574 case PR_O2_EXP_RSTS:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001575 memcpy(status_code, s->check.bi->data + 9, 3);
1576 memcpy(status_msg + strlen(status_msg) - 4, s->check.bi->data + 9, 3);
Willy Tarreaubd741542010-03-16 18:46:54 +01001577
1578 if ((s->proxy->options2 & PR_O2_EXP_TYPE) == PR_O2_EXP_STS)
1579 ret = strncmp(s->proxy->expect_str, status_code, 3) == 0;
1580 else
1581 ret = regexec(s->proxy->expect_regex, status_code, MAX_MATCH, pmatch, 0) == 0;
1582
1583 /* we necessarily have the response, so there are no partial failures */
1584 if (s->proxy->options2 & PR_O2_EXP_INV)
1585 ret = !ret;
1586
1587 set_server_check_status(s, ret ? HCHK_STATUS_L7OKD : HCHK_STATUS_L7STS, status_msg);
1588 break;
1589
1590 case PR_O2_EXP_STR:
1591 case PR_O2_EXP_RSTR:
1592 /* very simple response parser: ignore CR and only count consecutive LFs,
1593 * stop with contentptr pointing to first char after the double CRLF or
1594 * to '\0' if crlf < 2.
1595 */
1596 crlf = 0;
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001597 for (contentptr = s->check.bi->data; *contentptr; contentptr++) {
Willy Tarreaubd741542010-03-16 18:46:54 +01001598 if (crlf >= 2)
1599 break;
1600 if (*contentptr == '\r')
1601 continue;
1602 else if (*contentptr == '\n')
1603 crlf++;
1604 else
1605 crlf = 0;
1606 }
1607
1608 /* Check that response contains a body... */
1609 if (crlf < 2) {
1610 if (!done)
1611 return 0;
1612
1613 set_server_check_status(s, HCHK_STATUS_L7RSP,
1614 "HTTP content check could not find a response body");
1615 return 1;
1616 }
1617
1618 /* Check that response body is not empty... */
1619 if (*contentptr == '\0') {
Willy Tarreaua164fb52011-04-13 09:32:41 +02001620 if (!done)
1621 return 0;
1622
Willy Tarreaubd741542010-03-16 18:46:54 +01001623 set_server_check_status(s, HCHK_STATUS_L7RSP,
1624 "HTTP content check found empty response body");
1625 return 1;
1626 }
1627
1628 /* Check the response content against the supplied string
1629 * or regex... */
1630 if ((s->proxy->options2 & PR_O2_EXP_TYPE) == PR_O2_EXP_STR)
1631 ret = strstr(contentptr, s->proxy->expect_str) != NULL;
1632 else
1633 ret = regexec(s->proxy->expect_regex, contentptr, MAX_MATCH, pmatch, 0) == 0;
1634
1635 /* if we don't match, we may need to wait more */
1636 if (!ret && !done)
1637 return 0;
1638
1639 if (ret) {
1640 /* content matched */
1641 if (s->proxy->options2 & PR_O2_EXP_INV)
1642 set_server_check_status(s, HCHK_STATUS_L7RSP,
1643 "HTTP check matched unwanted content");
1644 else
1645 set_server_check_status(s, HCHK_STATUS_L7OKD,
1646 "HTTP content check matched");
1647 }
1648 else {
1649 if (s->proxy->options2 & PR_O2_EXP_INV)
1650 set_server_check_status(s, HCHK_STATUS_L7OKD,
1651 "HTTP check did not match unwanted content");
1652 else
1653 set_server_check_status(s, HCHK_STATUS_L7RSP,
1654 "HTTP content check did not match");
1655 }
1656 break;
1657 }
1658 return 1;
1659}
1660
1661/*
Willy Tarreaubaaee002006-06-26 02:48:02 +02001662 * Local variables:
1663 * c-indent-level: 8
1664 * c-basic-offset: 8
1665 * End:
1666 */