blob: c055017532323b8012b656206060bfc2a554b56a [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
2 * Health-checks functions.
3 *
Willy Tarreau26c25062009-03-08 09:38:41 +01004 * Copyright 2000-2009 Willy Tarreau <w@1wt.eu>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02005 * Copyright 2007-2009 Krzysztof Piotr Oledzki <ole@ans.pl>
Willy Tarreaubaaee002006-06-26 02:48:02 +02006 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 *
12 */
13
Willy Tarreaub8816082008-01-18 12:18:15 +010014#include <assert.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020015#include <ctype.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020016#include <errno.h>
17#include <fcntl.h>
18#include <stdio.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020019#include <stdlib.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020020#include <string.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020021#include <time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020022#include <unistd.h>
23#include <sys/socket.h>
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +040024#include <sys/types.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020025#include <netinet/in.h>
Willy Tarreau1274bc42009-07-15 07:16:31 +020026#include <netinet/tcp.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020027#include <arpa/inet.h>
28
Willy Tarreauc7e42382012-08-24 19:22:53 +020029#include <common/chunk.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020030#include <common/compat.h>
31#include <common/config.h>
32#include <common/mini-clist.h>
Willy Tarreau83749182007-04-15 20:56:27 +020033#include <common/standard.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020034#include <common/time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020035
36#include <types/global.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020037
38#include <proto/backend.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020039#include <proto/checks.h>
Simon Hormana2b9dad2013-02-12 10:45:54 +090040#include <proto/dumpstats.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020041#include <proto/fd.h>
42#include <proto/log.h>
43#include <proto/queue.h>
Willy Tarreauc6f4ce82009-06-10 11:09:37 +020044#include <proto/port_range.h>
Willy Tarreau3d300592007-03-18 18:34:41 +010045#include <proto/proto_http.h>
Willy Tarreaue8c66af2008-01-13 18:40:14 +010046#include <proto/proto_tcp.h>
Willy Tarreau2b5652f2006-12-31 17:46:05 +010047#include <proto/proxy.h>
Willy Tarreaufb56aab2012-09-28 14:40:02 +020048#include <proto/raw_sock.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020049#include <proto/server.h>
Simon Hormane0d1bfb2011-06-21 14:34:58 +090050#include <proto/session.h>
Willy Tarreau9e000c62011-03-10 14:03:36 +010051#include <proto/stream_interface.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020052#include <proto/task.h>
53
Willy Tarreaubd741542010-03-16 18:46:54 +010054static int httpchk_expect(struct server *s, int done);
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +020055static int tcpcheck_get_step_id(struct server *);
56static void tcpcheck_main(struct connection *);
Willy Tarreaubd741542010-03-16 18:46:54 +010057
Simon Horman63a4a822012-03-19 07:24:41 +090058static const struct check_status check_statuses[HCHK_STATUS_SIZE] = {
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010059 [HCHK_STATUS_UNKNOWN] = { CHK_RES_UNKNOWN, "UNK", "Unknown" },
60 [HCHK_STATUS_INI] = { CHK_RES_UNKNOWN, "INI", "Initializing" },
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020061 [HCHK_STATUS_START] = { /* SPECIAL STATUS*/ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020062
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010063 [HCHK_STATUS_HANA] = { CHK_RES_FAILED, "HANA", "Health analyze" },
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010064
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010065 [HCHK_STATUS_SOCKERR] = { CHK_RES_FAILED, "SOCKERR", "Socket error" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020066
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010067 [HCHK_STATUS_L4OK] = { CHK_RES_PASSED, "L4OK", "Layer4 check passed" },
68 [HCHK_STATUS_L4TOUT] = { CHK_RES_FAILED, "L4TOUT", "Layer4 timeout" },
69 [HCHK_STATUS_L4CON] = { CHK_RES_FAILED, "L4CON", "Layer4 connection problem" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020070
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010071 [HCHK_STATUS_L6OK] = { CHK_RES_PASSED, "L6OK", "Layer6 check passed" },
72 [HCHK_STATUS_L6TOUT] = { CHK_RES_FAILED, "L6TOUT", "Layer6 timeout" },
73 [HCHK_STATUS_L6RSP] = { CHK_RES_FAILED, "L6RSP", "Layer6 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020074
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010075 [HCHK_STATUS_L7TOUT] = { CHK_RES_FAILED, "L7TOUT", "Layer7 timeout" },
76 [HCHK_STATUS_L7RSP] = { CHK_RES_FAILED, "L7RSP", "Layer7 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020077
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020078 [HCHK_STATUS_L57DATA] = { /* DUMMY STATUS */ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020079
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010080 [HCHK_STATUS_L7OKD] = { CHK_RES_PASSED, "L7OK", "Layer7 check passed" },
81 [HCHK_STATUS_L7OKCD] = { CHK_RES_CONDPASS, "L7OKC", "Layer7 check conditionally passed" },
82 [HCHK_STATUS_L7STS] = { CHK_RES_FAILED, "L7STS", "Layer7 wrong status" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020083};
84
Simon Horman63a4a822012-03-19 07:24:41 +090085static const struct analyze_status analyze_statuses[HANA_STATUS_SIZE] = { /* 0: ignore, 1: error, 2: OK */
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010086 [HANA_STATUS_UNKNOWN] = { "Unknown", { 0, 0 }},
87
88 [HANA_STATUS_L4_OK] = { "L4 successful connection", { 2, 0 }},
89 [HANA_STATUS_L4_ERR] = { "L4 unsuccessful connection", { 1, 1 }},
90
91 [HANA_STATUS_HTTP_OK] = { "Correct http response", { 0, 2 }},
92 [HANA_STATUS_HTTP_STS] = { "Wrong http response", { 0, 1 }},
93 [HANA_STATUS_HTTP_HDRRSP] = { "Invalid http response (headers)", { 0, 1 }},
94 [HANA_STATUS_HTTP_RSP] = { "Invalid http response", { 0, 1 }},
95
96 [HANA_STATUS_HTTP_READ_ERROR] = { "Read error (http)", { 0, 1 }},
97 [HANA_STATUS_HTTP_READ_TIMEOUT] = { "Read timeout (http)", { 0, 1 }},
98 [HANA_STATUS_HTTP_BROKEN_PIPE] = { "Close from server (http)", { 0, 1 }},
99};
100
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200101/*
102 * Convert check_status code to description
103 */
104const char *get_check_status_description(short check_status) {
105
106 const char *desc;
107
108 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200109 desc = check_statuses[check_status].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200110 else
111 desc = NULL;
112
113 if (desc && *desc)
114 return desc;
115 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200116 return check_statuses[HCHK_STATUS_UNKNOWN].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200117}
118
119/*
120 * Convert check_status code to short info
121 */
122const char *get_check_status_info(short check_status) {
123
124 const char *info;
125
126 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200127 info = check_statuses[check_status].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200128 else
129 info = NULL;
130
131 if (info && *info)
132 return info;
133 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200134 return check_statuses[HCHK_STATUS_UNKNOWN].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200135}
136
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100137const char *get_analyze_status(short analyze_status) {
138
139 const char *desc;
140
141 if (analyze_status < HANA_STATUS_SIZE)
142 desc = analyze_statuses[analyze_status].desc;
143 else
144 desc = NULL;
145
146 if (desc && *desc)
147 return desc;
148 else
149 return analyze_statuses[HANA_STATUS_UNKNOWN].desc;
150}
151
Simon Horman4a741432013-02-23 15:35:38 +0900152static void server_status_printf(struct chunk *msg, struct server *s, struct check *check, int xferred) {
Willy Tarreau44267702011-10-28 15:35:33 +0200153 if (s->track)
Willy Tarreau77804732012-10-29 16:14:26 +0100154 chunk_appendf(msg, " via %s/%s",
Willy Tarreau44267702011-10-28 15:35:33 +0200155 s->track->proxy->id, s->track->id);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200156
Simon Horman4a741432013-02-23 15:35:38 +0900157 if (check) {
158 chunk_appendf(msg, ", reason: %s", get_check_status_description(check->status));
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200159
Simon Horman4a741432013-02-23 15:35:38 +0900160 if (check->status >= HCHK_STATUS_L57DATA)
161 chunk_appendf(msg, ", code: %d", check->code);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200162
Simon Horman4a741432013-02-23 15:35:38 +0900163 if (*check->desc) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200164 struct chunk src;
165
Willy Tarreau77804732012-10-29 16:14:26 +0100166 chunk_appendf(msg, ", info: \"");
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200167
Simon Horman4a741432013-02-23 15:35:38 +0900168 chunk_initlen(&src, check->desc, 0, strlen(check->desc));
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200169 chunk_asciiencode(msg, &src, '"');
170
Willy Tarreau77804732012-10-29 16:14:26 +0100171 chunk_appendf(msg, "\"");
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200172 }
173
Simon Horman4a741432013-02-23 15:35:38 +0900174 if (check->duration >= 0)
175 chunk_appendf(msg, ", check duration: %ldms", check->duration);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200176 }
177
Krzysztof Piotr Oledzki3bb05712010-09-27 13:10:50 +0200178 if (xferred >= 0) {
Krzysztof Piotr Oledzkib16a6072010-01-10 21:12:58 +0100179 if (!(s->state & SRV_RUNNING))
Willy Tarreau77804732012-10-29 16:14:26 +0100180 chunk_appendf(msg, ". %d active and %d backup servers left.%s"
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100181 " %d sessions active, %d requeued, %d remaining in queue",
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200182 s->proxy->srv_act, s->proxy->srv_bck,
183 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
184 s->cur_sess, xferred, s->nbpend);
185 else
Willy Tarreau77804732012-10-29 16:14:26 +0100186 chunk_appendf(msg, ". %d active and %d backup servers online.%s"
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100187 " %d sessions requeued, %d total in queue",
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200188 s->proxy->srv_act, s->proxy->srv_bck,
189 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
190 xferred, s->nbpend);
191 }
192}
193
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200194/*
Simon Horman4a741432013-02-23 15:35:38 +0900195 * Set check->status, update check->duration and fill check->result with
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100196 * an adequate CHK_RES_* value.
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200197 *
198 * Show information in logs about failed health check if server is UP
199 * or succeeded health checks if server is DOWN.
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200200 */
Simon Horman4a741432013-02-23 15:35:38 +0900201static void set_server_check_status(struct check *check, short status, const char *desc)
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100202{
Simon Horman4a741432013-02-23 15:35:38 +0900203 struct server *s = check->server;
204
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200205 if (status == HCHK_STATUS_START) {
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100206 check->result = CHK_RES_UNKNOWN; /* no result yet */
Simon Horman4a741432013-02-23 15:35:38 +0900207 check->desc[0] = '\0';
208 check->start = now;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200209 return;
210 }
211
Simon Horman4a741432013-02-23 15:35:38 +0900212 if (!check->status)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200213 return;
214
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200215 if (desc && *desc) {
Simon Horman4a741432013-02-23 15:35:38 +0900216 strncpy(check->desc, desc, HCHK_DESC_LEN-1);
217 check->desc[HCHK_DESC_LEN-1] = '\0';
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200218 } else
Simon Horman4a741432013-02-23 15:35:38 +0900219 check->desc[0] = '\0';
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200220
Simon Horman4a741432013-02-23 15:35:38 +0900221 check->status = status;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200222 if (check_statuses[status].result)
Simon Horman4a741432013-02-23 15:35:38 +0900223 check->result = check_statuses[status].result;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200224
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100225 if (status == HCHK_STATUS_HANA)
Simon Horman4a741432013-02-23 15:35:38 +0900226 check->duration = -1;
227 else if (!tv_iszero(&check->start)) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200228 /* set_server_check_status() may be called more than once */
Simon Horman4a741432013-02-23 15:35:38 +0900229 check->duration = tv_ms_elapsed(&check->start, &now);
230 tv_zero(&check->start);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200231 }
232
Simon Horman2f1f9552013-11-25 10:46:37 +0900233 /* Failure to connect to the agent as a secondary check should not
234 * cause the server to be marked down. So only log status changes
235 * for HCHK_STATUS_* statuses */
Willy Tarreau33434322013-12-11 21:15:19 +0100236 if ((check->state & CHK_ST_AGENT) && check->status < HCHK_STATUS_L7TOUT)
Simon Horman2f1f9552013-11-25 10:46:37 +0900237 return;
238
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200239 if (s->proxy->options2 & PR_O2_LOGHCHKS &&
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100240 (((check->health != 0) && (check->result == CHK_RES_FAILED)) ||
Simon Horman8c3d0be2013-11-25 10:46:40 +0900241 (((check->health != check->rise + check->fall - 1) ||
242 (!s->uweight && !(s->state & SRV_DRAIN)) ||
243 (s->uweight && (s->state & SRV_DRAIN))) &&
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100244 (check->result >= CHK_RES_PASSED)) ||
245 ((s->state & SRV_GOINGDOWN) && (check->result != CHK_RES_CONDPASS)) ||
246 (!(s->state & SRV_GOINGDOWN) && (check->result == CHK_RES_CONDPASS)))) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200247
248 int health, rise, fall, state;
249
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100250 chunk_reset(&trash);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200251
252 /* FIXME begin: calculate local version of the health/rise/fall/state */
Simon Horman125d0992013-02-24 17:23:38 +0900253 health = check->health;
Simon Horman58c32972013-11-25 10:46:38 +0900254 rise = check->rise;
255 fall = check->fall;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200256 state = s->state;
257
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100258 switch (check->result) {
259 case CHK_RES_FAILED:
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200260 if (health > rise) {
261 health--; /* still good */
262 } else {
263 if (health == rise)
264 state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
265
266 health = 0;
267 }
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100268 break;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200269
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100270 case CHK_RES_PASSED:
271 case CHK_RES_CONDPASS:
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200272 if (health < rise + fall - 1) {
273 health++; /* was bad, stays for a while */
274
275 if (health == rise)
276 state |= SRV_RUNNING;
277
278 if (health >= rise)
279 health = rise + fall - 1; /* OK now */
280 }
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100281
282 /* clear consecutive_errors if observing is enabled */
283 if (s->onerror)
284 s->consecutive_errors = 0;
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100285 break;
286 default:
287 break;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200288 }
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200289
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100290 chunk_appendf(&trash,
291 "Health check for %sserver %s/%s %s%s",
292 s->state & SRV_BACKUP ? "backup " : "",
293 s->proxy->id, s->id,
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100294 (check->result == CHK_RES_CONDPASS) ? "conditionally ":"",
295 (check->result >= CHK_RES_PASSED) ? "succeeded":"failed");
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200296
Simon Horman4a741432013-02-23 15:35:38 +0900297 server_status_printf(&trash, s, check, -1);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200298
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100299 chunk_appendf(&trash, ", status: %d/%d %s",
300 (state & SRV_RUNNING) ? (health - rise + 1) : (health),
301 (state & SRV_RUNNING) ? (fall) : (rise),
Simon Horman8c3d0be2013-11-25 10:46:40 +0900302 (state & SRV_RUNNING)?(s->eweight?"UP":"DRAIN"):"DOWN");
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200303
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100304 Warning("%s.\n", trash.str);
305 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200306 }
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200307}
308
Willy Tarreau48494c02007-11-30 10:41:39 +0100309/* sends a log message when a backend goes down, and also sets last
310 * change date.
311 */
312static void set_backend_down(struct proxy *be)
313{
314 be->last_change = now.tv_sec;
315 be->down_trans++;
316
317 Alert("%s '%s' has no server available!\n", proxy_type_str(be), be->id);
318 send_log(be, LOG_EMERG, "%s %s has no server available!\n", proxy_type_str(be), be->id);
319}
320
321/* Redistribute pending connections when a server goes down. The number of
322 * connections redistributed is returned.
323 */
324static int redistribute_pending(struct server *s)
325{
326 struct pendconn *pc, *pc_bck, *pc_end;
327 int xferred = 0;
328
329 FOREACH_ITEM_SAFE(pc, pc_bck, &s->pendconns, pc_end, struct pendconn *, list) {
330 struct session *sess = pc->sess;
Willy Tarreau4de91492010-01-22 19:10:05 +0100331 if ((sess->be->options & (PR_O_REDISP|PR_O_PERSIST)) == PR_O_REDISP &&
332 !(sess->flags & SN_FORCE_PRST)) {
Willy Tarreau48494c02007-11-30 10:41:39 +0100333 /* The REDISP option was specified. We will ignore
334 * cookie and force to balance or use the dispatcher.
335 */
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100336
Krzysztof Piotr Oledzki5a329cf2008-02-22 03:50:19 +0100337 /* it's left to the dispatcher to choose a server */
Willy Tarreau48494c02007-11-30 10:41:39 +0100338 sess->flags &= ~(SN_DIRECT | SN_ASSIGNED | SN_ADDR_SET);
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100339
Willy Tarreau48494c02007-11-30 10:41:39 +0100340 pendconn_free(pc);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200341 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100342 xferred++;
343 }
344 }
345 return xferred;
346}
347
348/* Check for pending connections at the backend, and assign some of them to
349 * the server coming up. The server's weight is checked before being assigned
350 * connections it may not be able to handle. The total number of transferred
351 * connections is returned.
352 */
353static int check_for_pending(struct server *s)
354{
355 int xferred;
356
357 if (!s->eweight)
358 return 0;
359
360 for (xferred = 0; !s->maxconn || xferred < srv_dynamic_maxconn(s); xferred++) {
361 struct session *sess;
362 struct pendconn *p;
363
364 p = pendconn_from_px(s->proxy);
365 if (!p)
366 break;
Willy Tarreau3fdb3662012-11-12 00:42:33 +0100367 p->sess->target = &s->obj_type;
Willy Tarreau48494c02007-11-30 10:41:39 +0100368 sess = p->sess;
369 pendconn_free(p);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200370 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100371 }
372 return xferred;
373}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200374
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700375/* Shutdown all connections of a server. The caller must pass a termination
376 * code in <why>, which must be one of SN_ERR_* indicating the reason for the
377 * shutdown.
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900378 */
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700379static void shutdown_sessions(struct server *srv, int why)
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900380{
381 struct session *session, *session_bck;
382
Willy Tarreaua2a64e92011-09-07 23:01:56 +0200383 list_for_each_entry_safe(session, session_bck, &srv->actconns, by_srv)
384 if (session->srv_conn == srv)
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700385 session_shutdown(session, why);
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900386}
387
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700388/* Shutdown all connections of all backup servers of a proxy. The caller must
389 * pass a termination code in <why>, which must be one of SN_ERR_* indicating
390 * the reason for the shutdown.
391 */
392static void shutdown_backup_sessions(struct proxy *px, int why)
393{
394 struct server *srv;
395
396 for (srv = px->srv; srv != NULL; srv = srv->next)
397 if (srv->state & SRV_BACKUP)
398 shutdown_sessions(srv, why);
399}
400
Willy Tarreaubaaee002006-06-26 02:48:02 +0200401/* Sets server <s> down, notifies by all available means, recounts the
402 * remaining servers on the proxy and transfers queued sessions whenever
Willy Tarreau5af3a692007-07-24 23:32:33 +0200403 * possible to other servers. It automatically recomputes the number of
404 * servers, but not the map.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200405 */
Simon Horman4a741432013-02-23 15:35:38 +0900406void set_server_down(struct check *check)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200407{
Simon Horman4a741432013-02-23 15:35:38 +0900408 struct server *s = check->server;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100409 struct server *srv;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200410 int xferred;
411
Cyril Bontécd19e512010-01-31 22:34:03 +0100412 if (s->state & SRV_MAINTAIN) {
Simon Horman58c32972013-11-25 10:46:38 +0900413 check->health = check->rise;
Cyril Bontécd19e512010-01-31 22:34:03 +0100414 }
415
Simon Horman58c32972013-11-25 10:46:38 +0900416 if ((s->state & SRV_RUNNING && check->health == check->rise) || s->track) {
Willy Tarreau48494c02007-11-30 10:41:39 +0100417 int srv_was_paused = s->state & SRV_GOINGDOWN;
Willy Tarreaud64d2252010-10-17 17:16:42 +0200418 int prev_srv_count = s->proxy->srv_bck + s->proxy->srv_act;
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200419
420 s->last_change = now.tv_sec;
Willy Tarreau48494c02007-11-30 10:41:39 +0100421 s->state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
Willy Tarreau9580d162012-05-19 19:07:40 +0200422 if (s->proxy->lbprm.set_server_status_down)
423 s->proxy->lbprm.set_server_status_down(s);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200424
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900425 if (s->onmarkeddown & HANA_ONMARKEDDOWN_SHUTDOWNSESSIONS)
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700426 shutdown_sessions(s, SN_ERR_DOWN);
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900427
Willy Tarreaubaaee002006-06-26 02:48:02 +0200428 /* we might have sessions queued on this server and waiting for
429 * a connection. Those which are redispatchable will be queued
430 * to another server or to the proxy itself.
431 */
Willy Tarreau48494c02007-11-30 10:41:39 +0100432 xferred = redistribute_pending(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100433
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100434 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100435
Cyril Bontécd19e512010-01-31 22:34:03 +0100436 if (s->state & SRV_MAINTAIN) {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100437 chunk_appendf(&trash,
438 "%sServer %s/%s is DOWN for maintenance", s->state & SRV_BACKUP ? "Backup " : "",
439 s->proxy->id, s->id);
Cyril Bontécd19e512010-01-31 22:34:03 +0100440 } else {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100441 chunk_appendf(&trash,
442 "%sServer %s/%s is DOWN", s->state & SRV_BACKUP ? "Backup " : "",
443 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100444
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100445 server_status_printf(&trash, s,
Simon Horman4a741432013-02-23 15:35:38 +0900446 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? check : 0),
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100447 xferred);
Cyril Bontécd19e512010-01-31 22:34:03 +0100448 }
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100449 Warning("%s.\n", trash.str);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200450
Willy Tarreau48494c02007-11-30 10:41:39 +0100451 /* we don't send an alert if the server was previously paused */
452 if (srv_was_paused)
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100453 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Willy Tarreau48494c02007-11-30 10:41:39 +0100454 else
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100455 send_log(s->proxy, LOG_ALERT, "%s.\n", trash.str);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200456
Willy Tarreaud64d2252010-10-17 17:16:42 +0200457 if (prev_srv_count && s->proxy->srv_bck == 0 && s->proxy->srv_act == 0)
Willy Tarreau48494c02007-11-30 10:41:39 +0100458 set_backend_down(s->proxy);
459
Krzysztof Piotr Oledzki052d4fd2009-10-04 14:52:57 +0200460 s->counters.down_trans++;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100461
Willy Tarreau1a53a3a2013-12-11 15:27:05 +0100462 for (srv = s->trackers; srv; srv = srv->tracknext)
463 if (!(srv->state & SRV_MAINTAIN))
464 /* Only notify tracking servers that are not already in maintenance. */
465 set_server_down(&srv->check);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200466 }
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100467
Simon Horman125d0992013-02-24 17:23:38 +0900468 check->health = 0; /* failure */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200469}
470
Simon Horman4a741432013-02-23 15:35:38 +0900471void set_server_up(struct check *check) {
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100472
Simon Horman4a741432013-02-23 15:35:38 +0900473 struct server *s = check->server;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100474 struct server *srv;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100475 int xferred;
Willy Tarreau45446782012-03-09 17:16:09 +0100476 unsigned int old_state = s->state;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100477
Cyril Bontécd19e512010-01-31 22:34:03 +0100478 if (s->state & SRV_MAINTAIN) {
Simon Horman58c32972013-11-25 10:46:38 +0900479 check->health = check->rise;
Cyril Bontécd19e512010-01-31 22:34:03 +0100480 }
481
Willy Tarreaubb9665e2013-12-14 16:14:15 +0100482 if (s->track ||
Willy Tarreau02541e82013-12-16 18:08:36 +0100483 ((s->check.state & CHK_ST_ENABLED) && (s->check.health == s->check.rise) &&
484 (s->agent.health >= s->agent.rise || !(s->agent.state & CHK_ST_ENABLED))) ||
485 ((s->agent.state & CHK_ST_ENABLED) && (s->agent.health == s->agent.rise) &&
Willy Tarreaue24d9632013-12-28 21:21:31 +0100486 (s->check.health >= s->check.rise || !(s->check.state & CHK_ST_ENABLED))) ||
487 (!(s->agent.state & CHK_ST_ENABLED) && !(s->check.state & CHK_ST_ENABLED))) {
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100488 if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0) {
489 if (s->proxy->last_change < now.tv_sec) // ignore negative times
490 s->proxy->down_time += now.tv_sec - s->proxy->last_change;
491 s->proxy->last_change = now.tv_sec;
492 }
493
494 if (s->last_change < now.tv_sec) // ignore negative times
495 s->down_time += now.tv_sec - s->last_change;
496
497 s->last_change = now.tv_sec;
498 s->state |= SRV_RUNNING;
Willy Tarreau45446782012-03-09 17:16:09 +0100499 s->state &= ~SRV_MAINTAIN;
Willy Tarreau33a08db2013-12-11 21:03:31 +0100500 s->check.state &= ~CHK_ST_PAUSED;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100501
502 if (s->slowstart > 0) {
503 s->state |= SRV_WARMINGUP;
Willy Tarreau2e993902011-10-31 11:53:20 +0100504 task_schedule(s->warmup, tick_add(now_ms, MS_TO_TICKS(MAX(1000, s->slowstart / 20))));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100505 }
Willy Tarreau004e0452013-11-21 11:22:01 +0100506
507 server_recalc_eweight(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100508
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700509 /* If the server is set with "on-marked-up shutdown-backup-sessions",
510 * and it's not a backup server and its effective weight is > 0,
511 * then it can accept new connections, so we shut down all sessions
512 * on all backup servers.
513 */
514 if ((s->onmarkedup & HANA_ONMARKEDUP_SHUTDOWNBACKUPSESSIONS) &&
515 !(s->state & SRV_BACKUP) && s->eweight)
516 shutdown_backup_sessions(s->proxy, SN_ERR_UP);
517
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100518 /* check if we can handle some connections queued at the proxy. We
519 * will take as many as we can handle.
520 */
521 xferred = check_for_pending(s);
522
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100523 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100524
Willy Tarreau45446782012-03-09 17:16:09 +0100525 if (old_state & SRV_MAINTAIN) {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100526 chunk_appendf(&trash,
527 "%sServer %s/%s is UP (leaving maintenance)", s->state & SRV_BACKUP ? "Backup " : "",
528 s->proxy->id, s->id);
Cyril Bontécd19e512010-01-31 22:34:03 +0100529 } else {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100530 chunk_appendf(&trash,
531 "%sServer %s/%s is UP", s->state & SRV_BACKUP ? "Backup " : "",
532 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100533
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100534 server_status_printf(&trash, s,
Simon Horman4a741432013-02-23 15:35:38 +0900535 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? check : NULL),
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100536 xferred);
Cyril Bontécd19e512010-01-31 22:34:03 +0100537 }
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100538
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100539 Warning("%s.\n", trash.str);
540 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100541
Willy Tarreau1a53a3a2013-12-11 15:27:05 +0100542 for (srv = s->trackers; srv; srv = srv->tracknext)
543 if (!(srv->state & SRV_MAINTAIN))
544 /* Only notify tracking servers if they're not in maintenance. */
545 set_server_up(&srv->check);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100546 }
547
Simon Horman58c32972013-11-25 10:46:38 +0900548 if (check->health >= check->rise)
549 check->health = check->rise + check->fall - 1; /* OK now */
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100550
551}
552
Simon Horman4a741432013-02-23 15:35:38 +0900553static void set_server_disabled(struct check *check) {
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100554
Simon Horman4a741432013-02-23 15:35:38 +0900555 struct server *s = check->server;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100556 struct server *srv;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100557 int xferred;
558
559 s->state |= SRV_GOINGDOWN;
Willy Tarreau9580d162012-05-19 19:07:40 +0200560 if (s->proxy->lbprm.set_server_status_down)
561 s->proxy->lbprm.set_server_status_down(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100562
563 /* we might have sessions queued on this server and waiting for
564 * a connection. Those which are redispatchable will be queued
565 * to another server or to the proxy itself.
566 */
567 xferred = redistribute_pending(s);
568
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100569 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100570
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100571 chunk_appendf(&trash,
572 "Load-balancing on %sServer %s/%s is disabled",
573 s->state & SRV_BACKUP ? "Backup " : "",
574 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100575
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100576 server_status_printf(&trash, s,
Simon Horman4a741432013-02-23 15:35:38 +0900577 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? check : NULL),
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100578 xferred);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100579
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100580 Warning("%s.\n", trash.str);
581 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100582
583 if (!s->proxy->srv_bck && !s->proxy->srv_act)
584 set_backend_down(s->proxy);
585
Willy Tarreau1a53a3a2013-12-11 15:27:05 +0100586 for (srv = s->trackers; srv; srv = srv->tracknext)
587 set_server_disabled(&srv->check);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100588}
589
Simon Horman4a741432013-02-23 15:35:38 +0900590static void set_server_enabled(struct check *check) {
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100591
Simon Horman4a741432013-02-23 15:35:38 +0900592 struct server *s = check->server;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100593 struct server *srv;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100594 int xferred;
595
596 s->state &= ~SRV_GOINGDOWN;
Willy Tarreau9580d162012-05-19 19:07:40 +0200597 if (s->proxy->lbprm.set_server_status_up)
598 s->proxy->lbprm.set_server_status_up(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100599
600 /* check if we can handle some connections queued at the proxy. We
601 * will take as many as we can handle.
602 */
603 xferred = check_for_pending(s);
604
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100605 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100606
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100607 chunk_appendf(&trash,
608 "Load-balancing on %sServer %s/%s is enabled again",
609 s->state & SRV_BACKUP ? "Backup " : "",
610 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100611
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100612 server_status_printf(&trash, s,
Simon Horman4a741432013-02-23 15:35:38 +0900613 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? check : NULL),
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100614 xferred);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100615
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100616 Warning("%s.\n", trash.str);
617 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100618
Willy Tarreau1a53a3a2013-12-11 15:27:05 +0100619 for (srv = s->trackers; srv; srv = srv->tracknext)
620 set_server_enabled(&srv->check);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100621}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200622
Simon Hormand8583062013-11-25 10:46:33 +0900623static void check_failed(struct check *check)
624{
625 struct server *s = check->server;
626
Simon Horman2f1f9552013-11-25 10:46:37 +0900627 /* The agent secondary check should only cause a server to be marked
628 * as down if check->status is HCHK_STATUS_L7STS, which indicates
629 * that the agent returned "fail", "stopped" or "down".
630 * The implication here is that failure to connect to the agent
631 * as a secondary check should not cause the server to be marked
632 * down. */
Willy Tarreau33434322013-12-11 21:15:19 +0100633 if ((check->state & CHK_ST_AGENT) && check->status != HCHK_STATUS_L7STS)
Simon Horman2f1f9552013-11-25 10:46:37 +0900634 return;
635
Simon Horman58c32972013-11-25 10:46:38 +0900636 if (check->health > check->rise) {
Simon Hormand8583062013-11-25 10:46:33 +0900637 check->health--; /* still good */
638 s->counters.failed_checks++;
639 }
640 else
641 set_server_down(check);
642}
643
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100644void health_adjust(struct server *s, short status)
645{
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100646 int failed;
647 int expire;
648
649 /* return now if observing nor health check is not enabled */
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200650 if (!s->observe || !s->check.task)
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100651 return;
652
653 if (s->observe >= HANA_OBS_SIZE)
654 return;
655
Willy Tarreaubb956662013-01-24 00:37:39 +0100656 if (status >= HANA_STATUS_SIZE || !analyze_statuses[status].desc)
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100657 return;
658
659 switch (analyze_statuses[status].lr[s->observe - 1]) {
660 case 1:
661 failed = 1;
662 break;
663
664 case 2:
665 failed = 0;
666 break;
667
668 default:
669 return;
670 }
671
672 if (!failed) {
673 /* good: clear consecutive_errors */
674 s->consecutive_errors = 0;
675 return;
676 }
677
678 s->consecutive_errors++;
679
680 if (s->consecutive_errors < s->consecutive_errors_limit)
681 return;
682
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100683 chunk_printf(&trash, "Detected %d consecutive errors, last one was: %s",
684 s->consecutive_errors, get_analyze_status(status));
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100685
686 switch (s->onerror) {
687 case HANA_ONERR_FASTINTER:
688 /* force fastinter - nothing to do here as all modes force it */
689 break;
690
691 case HANA_ONERR_SUDDTH:
692 /* simulate a pre-fatal failed health check */
Simon Horman58c32972013-11-25 10:46:38 +0900693 if (s->check.health > s->check.rise)
694 s->check.health = s->check.rise + 1;
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100695
696 /* no break - fall through */
697
698 case HANA_ONERR_FAILCHK:
699 /* simulate a failed health check */
Simon Horman4a741432013-02-23 15:35:38 +0900700 set_server_check_status(&s->check, HCHK_STATUS_HANA, trash.str);
Simon Hormand8583062013-11-25 10:46:33 +0900701 check_failed(&s->check);
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100702
703 break;
704
705 case HANA_ONERR_MARKDWN:
706 /* mark server down */
Simon Horman58c32972013-11-25 10:46:38 +0900707 s->check.health = s->check.rise;
Simon Horman4a741432013-02-23 15:35:38 +0900708 set_server_check_status(&s->check, HCHK_STATUS_HANA, trash.str);
709 set_server_down(&s->check);
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100710
711 break;
712
713 default:
714 /* write a warning? */
715 break;
716 }
717
718 s->consecutive_errors = 0;
719 s->counters.failed_hana++;
720
Simon Horman66183002013-02-23 10:16:43 +0900721 if (s->check.fastinter) {
722 expire = tick_add(now_ms, MS_TO_TICKS(s->check.fastinter));
Sergiy Prykhodko1d57e502013-09-21 12:05:00 +0300723 if (s->check.task->expire > expire) {
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200724 s->check.task->expire = expire;
Sergiy Prykhodko1d57e502013-09-21 12:05:00 +0300725 /* requeue check task with new expire */
726 task_queue(s->check.task);
727 }
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100728 }
729}
730
Willy Tarreauef781042010-01-27 11:53:01 +0100731static int httpchk_build_status_header(struct server *s, char *buffer)
732{
733 int sv_state;
734 int ratio;
735 int hlen = 0;
736 const char *srv_hlt_st[7] = { "DOWN", "DOWN %d/%d",
737 "UP %d/%d", "UP",
738 "NOLB %d/%d", "NOLB",
739 "no check" };
740
741 memcpy(buffer + hlen, "X-Haproxy-Server-State: ", 24);
742 hlen += 24;
743
Willy Tarreauff5ae352013-12-11 20:36:34 +0100744 if (!(s->check.state & CHK_ST_ENABLED))
745 sv_state = 6;
Willy Tarreauef781042010-01-27 11:53:01 +0100746 else if (s->state & SRV_RUNNING) {
Simon Horman58c32972013-11-25 10:46:38 +0900747 if (s->check.health == s->check.rise + s->check.fall - 1)
Willy Tarreauef781042010-01-27 11:53:01 +0100748 sv_state = 3; /* UP */
749 else
750 sv_state = 2; /* going down */
751
752 if (s->state & SRV_GOINGDOWN)
753 sv_state += 2;
754 } else {
Simon Horman125d0992013-02-24 17:23:38 +0900755 if (s->check.health)
Willy Tarreauef781042010-01-27 11:53:01 +0100756 sv_state = 1; /* going up */
757 else
758 sv_state = 0; /* DOWN */
759 }
760
761 hlen += sprintf(buffer + hlen,
762 srv_hlt_st[sv_state],
Simon Horman58c32972013-11-25 10:46:38 +0900763 (s->state & SRV_RUNNING) ? (s->check.health - s->check.rise + 1) : (s->check.health),
764 (s->state & SRV_RUNNING) ? (s->check.fall) : (s->check.rise));
Willy Tarreauef781042010-01-27 11:53:01 +0100765
766 hlen += sprintf(buffer + hlen, "; name=%s/%s; node=%s; weight=%d/%d; scur=%d/%d; qcur=%d",
767 s->proxy->id, s->id,
768 global.node,
769 (s->eweight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv,
770 (s->proxy->lbprm.tot_weight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv,
771 s->cur_sess, s->proxy->beconn - s->proxy->nbpend,
772 s->nbpend);
773
774 if ((s->state & SRV_WARMINGUP) &&
775 now.tv_sec < s->last_change + s->slowstart &&
776 now.tv_sec >= s->last_change) {
777 ratio = MAX(1, 100 * (now.tv_sec - s->last_change) / s->slowstart);
778 hlen += sprintf(buffer + hlen, "; throttle=%d%%", ratio);
779 }
780
781 buffer[hlen++] = '\r';
782 buffer[hlen++] = '\n';
783
784 return hlen;
785}
786
Willy Tarreau20a18342013-12-05 00:31:46 +0100787/* Check the connection. If an error has already been reported or the socket is
788 * closed, keep errno intact as it is supposed to contain the valid error code.
789 * If no error is reported, check the socket's error queue using getsockopt().
790 * Warning, this must be done only once when returning from poll, and never
791 * after an I/O error was attempted, otherwise the error queue might contain
792 * inconsistent errors. If an error is detected, the CO_FL_ERROR is set on the
793 * socket. Returns non-zero if an error was reported, zero if everything is
794 * clean (including a properly closed socket).
795 */
796static int retrieve_errno_from_socket(struct connection *conn)
797{
798 int skerr;
799 socklen_t lskerr = sizeof(skerr);
800
801 if (conn->flags & CO_FL_ERROR && ((errno && errno != EAGAIN) || !conn->ctrl))
802 return 1;
803
Willy Tarreauf79c8172013-10-21 16:30:56 +0200804 if (!(conn->flags & CO_FL_CTRL_READY) || !conn->ctrl)
Willy Tarreau20a18342013-12-05 00:31:46 +0100805 return 0;
806
807 if (getsockopt(conn->t.sock.fd, SOL_SOCKET, SO_ERROR, &skerr, &lskerr) == 0)
808 errno = skerr;
809
810 if (errno == EAGAIN)
811 errno = 0;
812
813 if (!errno) {
814 /* we could not retrieve an error, that does not mean there is
815 * none. Just don't change anything and only report the prior
816 * error if any.
817 */
818 if (conn->flags & CO_FL_ERROR)
819 return 1;
820 else
821 return 0;
822 }
823
824 conn->flags |= CO_FL_ERROR | CO_FL_SOCK_WR_SH | CO_FL_SOCK_RD_SH;
825 return 1;
826}
827
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100828/* Try to collect as much information as possible on the connection status,
829 * and adjust the server status accordingly. It may make use of <errno_bck>
830 * if non-null when the caller is absolutely certain of its validity (eg:
831 * checked just after a syscall). If the caller doesn't have a valid errno,
832 * it can pass zero, and retrieve_errno_from_socket() will be called to try
833 * to extract errno from the socket. If no error is reported, it will consider
834 * the <expired> flag. This is intended to be used when a connection error was
835 * reported in conn->flags or when a timeout was reported in <expired>. The
836 * function takes care of not updating a server status which was already set.
837 * All situations where at least one of <expired> or CO_FL_ERROR are set
838 * produce a status.
839 */
840static void chk_report_conn_err(struct connection *conn, int errno_bck, int expired)
841{
842 struct check *check = conn->owner;
843 const char *err_msg;
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200844 struct chunk *chk;
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100845
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100846 if (check->result != CHK_RES_UNKNOWN)
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100847 return;
848
849 errno = errno_bck;
850 if (!errno || errno == EAGAIN)
851 retrieve_errno_from_socket(conn);
852
853 if (!(conn->flags & CO_FL_ERROR) && !expired)
854 return;
855
856 /* we'll try to build a meaningful error message depending on the
857 * context of the error possibly present in conn->err_code, and the
858 * socket error possibly collected above. This is useful to know the
859 * exact step of the L6 layer (eg: SSL handshake).
860 */
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200861 chk = get_trash_chunk();
862
863 if (check->type == PR_O2_TCPCHK_CHK) {
864 chunk_printf(chk, " at step %d of tcp-check", tcpcheck_get_step_id(check->server));
865 /* we were looking for a string */
866 if (check->current_step && check->current_step->action == TCPCHK_ACT_EXPECT) {
867 if (check->current_step->string)
868 chunk_appendf(chk, " (string '%s')", check->current_step->string);
869 else if (check->current_step->expect_regex)
870 chunk_appendf(chk, " (expect regex)");
871 }
872 else if (check->current_step && check->current_step->action == TCPCHK_ACT_SEND) {
873 chunk_appendf(chk, " (send)");
874 }
875 }
876
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100877 if (conn->err_code) {
878 if (errno && errno != EAGAIN)
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200879 chunk_printf(&trash, "%s (%s)%s", conn_err_code_str(conn), strerror(errno), chk->str);
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100880 else
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200881 chunk_printf(&trash, "%s%s", conn_err_code_str(conn), chk->str);
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100882 err_msg = trash.str;
883 }
884 else {
885 if (errno && errno != EAGAIN) {
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200886 chunk_printf(&trash, "%s%s", strerror(errno), chk->str);
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100887 err_msg = trash.str;
888 }
889 else {
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200890 err_msg = chk->str;
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100891 }
892 }
893
894 if ((conn->flags & (CO_FL_CONNECTED|CO_FL_WAIT_L4_CONN)) == CO_FL_WAIT_L4_CONN) {
895 /* L4 not established (yet) */
896 if (conn->flags & CO_FL_ERROR)
897 set_server_check_status(check, HCHK_STATUS_L4CON, err_msg);
898 else if (expired)
899 set_server_check_status(check, HCHK_STATUS_L4TOUT, err_msg);
900 }
901 else if ((conn->flags & (CO_FL_CONNECTED|CO_FL_WAIT_L6_CONN)) == CO_FL_WAIT_L6_CONN) {
902 /* L6 not established (yet) */
903 if (conn->flags & CO_FL_ERROR)
904 set_server_check_status(check, HCHK_STATUS_L6RSP, err_msg);
905 else if (expired)
906 set_server_check_status(check, HCHK_STATUS_L6TOUT, err_msg);
907 }
908 else if (conn->flags & CO_FL_ERROR) {
909 /* I/O error after connection was established and before we could diagnose */
910 set_server_check_status(check, HCHK_STATUS_SOCKERR, err_msg);
911 }
912 else if (expired) {
913 /* connection established but expired check */
914 if (check->type == PR_O2_SSL3_CHK)
915 set_server_check_status(check, HCHK_STATUS_L6TOUT, err_msg);
916 else /* HTTP, SMTP, ... */
917 set_server_check_status(check, HCHK_STATUS_L7TOUT, err_msg);
918 }
919
920 return;
921}
922
Willy Tarreaubaaee002006-06-26 02:48:02 +0200923/*
924 * This function is used only for server health-checks. It handles
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200925 * the connection acknowledgement. If the proxy requires L7 health-checks,
926 * it sends the request. In other cases, it calls set_server_check_status()
Simon Horman4a741432013-02-23 15:35:38 +0900927 * to set check->status, check->duration and check->result.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200928 */
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200929static void event_srv_chk_w(struct connection *conn)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200930{
Simon Horman4a741432013-02-23 15:35:38 +0900931 struct check *check = conn->owner;
932 struct server *s = check->server;
Simon Horman4a741432013-02-23 15:35:38 +0900933 struct task *t = check->task;
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200934
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100935 if (unlikely(check->result == CHK_RES_FAILED))
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100936 goto out_wakeup;
937
938 if (conn->flags & (CO_FL_HANDSHAKE | CO_FL_WAIT_WR))
939 return;
940
Willy Tarreau20a18342013-12-05 00:31:46 +0100941 if (retrieve_errno_from_socket(conn)) {
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100942 chk_report_conn_err(conn, errno, 0);
Willy Tarreau20a18342013-12-05 00:31:46 +0100943 __conn_data_stop_both(conn);
944 goto out_wakeup;
945 }
Krzysztof Piotr Oledzki6492db52010-01-02 22:03:01 +0100946
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100947 if (conn->flags & (CO_FL_SOCK_WR_SH | CO_FL_DATA_WR_SH)) {
Willy Tarreau20a18342013-12-05 00:31:46 +0100948 /* if the output is closed, we can't do anything */
949 conn->flags |= CO_FL_ERROR;
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100950 chk_report_conn_err(conn, 0, 0);
Willy Tarreau20a18342013-12-05 00:31:46 +0100951 goto out_wakeup;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200952 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200953
Willy Tarreau06559ac2013-12-05 01:53:08 +0100954 /* here, we know that the connection is established. That's enough for
955 * a pure TCP check.
956 */
957 if (!check->type)
958 goto out_wakeup;
959
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200960 if (check->type == PR_O2_TCPCHK_CHK) {
961 tcpcheck_main(conn);
962 return;
963 }
964
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100965 if (check->bo->o) {
966 conn->xprt->snd_buf(conn, check->bo, MSG_DONTWAIT | MSG_NOSIGNAL);
967 if (conn->flags & CO_FL_ERROR) {
968 chk_report_conn_err(conn, errno, 0);
969 __conn_data_stop_both(conn);
970 goto out_wakeup;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200971 }
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100972 if (check->bo->o)
973 return;
974 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200975
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100976 /* full request sent, we allow up to <timeout.check> if nonzero for a response */
977 if (s->proxy->timeout.check) {
978 t->expire = tick_add_ifset(now_ms, s->proxy->timeout.check);
979 task_queue(t);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200980 }
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100981 goto out_nowake;
982
Willy Tarreau83749182007-04-15 20:56:27 +0200983 out_wakeup:
Willy Tarreaufdccded2008-08-29 18:19:04 +0200984 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreau83749182007-04-15 20:56:27 +0200985 out_nowake:
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200986 __conn_data_stop_send(conn); /* nothing more to write */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200987}
988
Willy Tarreaubaaee002006-06-26 02:48:02 +0200989/*
Willy Tarreauf3c69202006-07-09 16:42:34 +0200990 * This function is used only for server health-checks. It handles the server's
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +0200991 * reply to an HTTP request, SSL HELLO or MySQL client Auth. It calls
Simon Horman4a741432013-02-23 15:35:38 +0900992 * set_server_check_status() to update check->status, check->duration
993 * and check->result.
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200994
995 * The set_server_check_status function is called with HCHK_STATUS_L7OKD if
996 * an HTTP server replies HTTP 2xx or 3xx (valid responses), if an SMTP server
997 * returns 2xx, HCHK_STATUS_L6OK if an SSL server returns at least 5 bytes in
998 * response to an SSL HELLO (the principle is that this is enough to
999 * distinguish between an SSL server and a pure TCP relay). All other cases will
1000 * call it with a proper error status like HCHK_STATUS_L7STS, HCHK_STATUS_L6RSP,
1001 * etc.
Willy Tarreaubaaee002006-06-26 02:48:02 +02001002 */
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001003static void event_srv_chk_r(struct connection *conn)
Willy Tarreaubaaee002006-06-26 02:48:02 +02001004{
Simon Horman4a741432013-02-23 15:35:38 +09001005 struct check *check = conn->owner;
1006 struct server *s = check->server;
1007 struct task *t = check->task;
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001008 char *desc;
Willy Tarreau03938182010-03-17 21:52:07 +01001009 int done;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001010 unsigned short msglen;
Willy Tarreau83749182007-04-15 20:56:27 +02001011
Willy Tarreau6aaa1b82013-12-11 17:09:34 +01001012 if (unlikely(check->result == CHK_RES_FAILED))
Willy Tarreau83749182007-04-15 20:56:27 +02001013 goto out_wakeup;
Willy Tarreau83749182007-04-15 20:56:27 +02001014
Willy Tarreau5a78f362012-11-23 12:47:05 +01001015 if (conn->flags & (CO_FL_HANDSHAKE | CO_FL_WAIT_RD))
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001016 return;
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02001017
1018 if (check->type == PR_O2_TCPCHK_CHK) {
1019 tcpcheck_main(conn);
1020 return;
1021 }
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001022
Willy Tarreau83749182007-04-15 20:56:27 +02001023 /* Warning! Linux returns EAGAIN on SO_ERROR if data are still available
1024 * but the connection was closed on the remote end. Fortunately, recv still
1025 * works correctly and we don't need to do the getsockopt() on linux.
1026 */
Nick Chalk57b1bf72010-03-16 15:50:46 +00001027
1028 /* Set buffer to point to the end of the data already read, and check
1029 * that there is free space remaining. If the buffer is full, proceed
1030 * with running the checks without attempting another socket read.
1031 */
Nick Chalk57b1bf72010-03-16 15:50:46 +00001032
Willy Tarreau03938182010-03-17 21:52:07 +01001033 done = 0;
Nick Chalk57b1bf72010-03-16 15:50:46 +00001034
Simon Horman4a741432013-02-23 15:35:38 +09001035 conn->xprt->rcv_buf(conn, check->bi, check->bi->size);
Willy Tarreauf1503172012-09-28 19:39:36 +02001036 if (conn->flags & (CO_FL_ERROR | CO_FL_SOCK_RD_SH | CO_FL_DATA_RD_SH)) {
Willy Tarreau03938182010-03-17 21:52:07 +01001037 done = 1;
Simon Horman4a741432013-02-23 15:35:38 +09001038 if ((conn->flags & CO_FL_ERROR) && !check->bi->i) {
Willy Tarreauf1503172012-09-28 19:39:36 +02001039 /* Report network errors only if we got no other data. Otherwise
1040 * we'll let the upper layers decide whether the response is OK
1041 * or not. It is very common that an RST sent by the server is
1042 * reported as an error just after the last data chunk.
1043 */
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001044 chk_report_conn_err(conn, errno, 0);
Willy Tarreauc1a07962010-03-16 20:55:43 +01001045 goto out_wakeup;
1046 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001047 }
1048
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001049
Willy Tarreau03938182010-03-17 21:52:07 +01001050 /* Intermediate or complete response received.
Simon Horman4a741432013-02-23 15:35:38 +09001051 * Terminate string in check->bi->data buffer.
Willy Tarreau03938182010-03-17 21:52:07 +01001052 */
Simon Horman4a741432013-02-23 15:35:38 +09001053 if (check->bi->i < check->bi->size)
1054 check->bi->data[check->bi->i] = '\0';
Willy Tarreau03938182010-03-17 21:52:07 +01001055 else {
Simon Horman4a741432013-02-23 15:35:38 +09001056 check->bi->data[check->bi->i - 1] = '\0';
Willy Tarreau03938182010-03-17 21:52:07 +01001057 done = 1; /* buffer full, don't wait for more data */
1058 }
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001059
Nick Chalk57b1bf72010-03-16 15:50:46 +00001060 /* Run the checks... */
Simon Horman4a741432013-02-23 15:35:38 +09001061 switch (check->type) {
Willy Tarreau1620ec32011-08-06 17:05:02 +02001062 case PR_O2_HTTP_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001063 if (!done && check->bi->i < strlen("HTTP/1.0 000\r"))
Willy Tarreau03938182010-03-17 21:52:07 +01001064 goto wait_more_data;
1065
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001066 /* Check if the server speaks HTTP 1.X */
Simon Horman4a741432013-02-23 15:35:38 +09001067 if ((check->bi->i < strlen("HTTP/1.0 000\r")) ||
1068 (memcmp(check->bi->data, "HTTP/1.", 7) != 0 ||
1069 (*(check->bi->data + 12) != ' ' && *(check->bi->data + 12) != '\r')) ||
1070 !isdigit((unsigned char) *(check->bi->data + 9)) || !isdigit((unsigned char) *(check->bi->data + 10)) ||
1071 !isdigit((unsigned char) *(check->bi->data + 11))) {
1072 cut_crlf(check->bi->data);
1073 set_server_check_status(check, HCHK_STATUS_L7RSP, check->bi->data);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001074
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001075 goto out_wakeup;
1076 }
1077
Simon Horman4a741432013-02-23 15:35:38 +09001078 check->code = str2uic(check->bi->data + 9);
1079 desc = ltrim(check->bi->data + 12, ' ');
Nick Chalk57b1bf72010-03-16 15:50:46 +00001080
Willy Tarreaubd741542010-03-16 18:46:54 +01001081 if ((s->proxy->options & PR_O_DISABLE404) &&
Simon Horman4a741432013-02-23 15:35:38 +09001082 (s->state & SRV_RUNNING) && (check->code == 404)) {
Nick Chalk57b1bf72010-03-16 15:50:46 +00001083 /* 404 may be accepted as "stopping" only if the server was up */
1084 cut_crlf(desc);
Simon Horman4a741432013-02-23 15:35:38 +09001085 set_server_check_status(check, HCHK_STATUS_L7OKCD, desc);
Nick Chalk57b1bf72010-03-16 15:50:46 +00001086 }
Willy Tarreaubd741542010-03-16 18:46:54 +01001087 else if (s->proxy->options2 & PR_O2_EXP_TYPE) {
1088 /* Run content verification check... We know we have at least 13 chars */
1089 if (!httpchk_expect(s, done))
1090 goto wait_more_data;
1091 }
1092 /* check the reply : HTTP/1.X 2xx and 3xx are OK */
Simon Horman4a741432013-02-23 15:35:38 +09001093 else if (*(check->bi->data + 9) == '2' || *(check->bi->data + 9) == '3') {
Willy Tarreaubd741542010-03-16 18:46:54 +01001094 cut_crlf(desc);
Simon Horman4a741432013-02-23 15:35:38 +09001095 set_server_check_status(check, HCHK_STATUS_L7OKD, desc);
Willy Tarreaubd741542010-03-16 18:46:54 +01001096 }
Nick Chalk57b1bf72010-03-16 15:50:46 +00001097 else {
1098 cut_crlf(desc);
Simon Horman4a741432013-02-23 15:35:38 +09001099 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Nick Chalk57b1bf72010-03-16 15:50:46 +00001100 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001101 break;
1102
1103 case PR_O2_SSL3_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001104 if (!done && check->bi->i < 5)
Willy Tarreau03938182010-03-17 21:52:07 +01001105 goto wait_more_data;
1106
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001107 /* Check for SSLv3 alert or handshake */
Simon Horman4a741432013-02-23 15:35:38 +09001108 if ((check->bi->i >= 5) && (*check->bi->data == 0x15 || *check->bi->data == 0x16))
1109 set_server_check_status(check, HCHK_STATUS_L6OK, NULL);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +02001110 else
Simon Horman4a741432013-02-23 15:35:38 +09001111 set_server_check_status(check, HCHK_STATUS_L6RSP, NULL);
Willy Tarreau1620ec32011-08-06 17:05:02 +02001112 break;
1113
1114 case PR_O2_SMTP_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001115 if (!done && check->bi->i < strlen("000\r"))
Willy Tarreau03938182010-03-17 21:52:07 +01001116 goto wait_more_data;
1117
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001118 /* Check if the server speaks SMTP */
Simon Horman4a741432013-02-23 15:35:38 +09001119 if ((check->bi->i < strlen("000\r")) ||
1120 (*(check->bi->data + 3) != ' ' && *(check->bi->data + 3) != '\r') ||
1121 !isdigit((unsigned char) *check->bi->data) || !isdigit((unsigned char) *(check->bi->data + 1)) ||
1122 !isdigit((unsigned char) *(check->bi->data + 2))) {
1123 cut_crlf(check->bi->data);
1124 set_server_check_status(check, HCHK_STATUS_L7RSP, check->bi->data);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001125
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001126 goto out_wakeup;
1127 }
1128
Simon Horman4a741432013-02-23 15:35:38 +09001129 check->code = str2uic(check->bi->data);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001130
Simon Horman4a741432013-02-23 15:35:38 +09001131 desc = ltrim(check->bi->data + 3, ' ');
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +02001132 cut_crlf(desc);
1133
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001134 /* Check for SMTP code 2xx (should be 250) */
Simon Horman4a741432013-02-23 15:35:38 +09001135 if (*check->bi->data == '2')
1136 set_server_check_status(check, HCHK_STATUS_L7OKD, desc);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +02001137 else
Simon Horman4a741432013-02-23 15:35:38 +09001138 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Willy Tarreau1620ec32011-08-06 17:05:02 +02001139 break;
1140
Simon Hormana2b9dad2013-02-12 10:45:54 +09001141 case PR_O2_LB_AGENT_CHK: {
1142 short status = HCHK_STATUS_L7RSP;
1143 const char *desc = "Unknown feedback string";
1144 const char *down_cmd = NULL;
Simon Horman671b6f02013-11-25 10:46:39 +09001145 int disabled;
Willy Tarreau9809b782013-12-11 21:40:11 +01001146 char *p;
Simon Hormana2b9dad2013-02-12 10:45:54 +09001147
Willy Tarreau9809b782013-12-11 21:40:11 +01001148 /* get a complete line first */
1149 p = check->bi->data;
1150 while (*p && *p != '\n' && *p != '\r')
1151 p++;
1152
1153 if (!*p) {
1154 if (!done)
1155 goto wait_more_data;
Simon Hormana2b9dad2013-02-12 10:45:54 +09001156
Willy Tarreau9809b782013-12-11 21:40:11 +01001157 /* at least inform the admin that the agent is mis-behaving */
1158 set_server_check_status(check, check->status, "Ignoring incomplete line from agent");
1159 break;
1160 }
1161 *p = 0;
Simon Hormana2b9dad2013-02-12 10:45:54 +09001162
Simon Horman671b6f02013-11-25 10:46:39 +09001163 /*
1164 * The agent may have been disabled after a check was
1165 * initialised. If so, ignore weight changes and drain
1166 * settings from the agent. Note that the setting is
1167 * always present in the state of the agent the server,
1168 * regardless of if the agent is being run as a primary or
1169 * secondary check. That is, regardless of if the check
1170 * parameter of this function is the agent or check field
1171 * of the server.
1172 */
Willy Tarreau2e10f5a2013-12-11 20:11:55 +01001173 disabled = !(check->server->agent.state & CHK_ST_ENABLED);
Simon Horman671b6f02013-11-25 10:46:39 +09001174
Simon Horman4a741432013-02-23 15:35:38 +09001175 if (strchr(check->bi->data, '%')) {
Simon Horman671b6f02013-11-25 10:46:39 +09001176 if (disabled)
1177 break;
Simon Horman4a741432013-02-23 15:35:38 +09001178 desc = server_parse_weight_change_request(s, check->bi->data);
Simon Hormana2b9dad2013-02-12 10:45:54 +09001179 if (!desc) {
1180 status = HCHK_STATUS_L7OKD;
Simon Horman4a741432013-02-23 15:35:38 +09001181 desc = check->bi->data;
Simon Hormana2b9dad2013-02-12 10:45:54 +09001182 }
Simon Horman4a741432013-02-23 15:35:38 +09001183 } else if (!strcasecmp(check->bi->data, "drain")) {
Simon Horman671b6f02013-11-25 10:46:39 +09001184 if (disabled)
1185 break;
Simon Hormana2b9dad2013-02-12 10:45:54 +09001186 desc = server_parse_weight_change_request(s, "0%");
1187 if (!desc) {
1188 desc = "drain";
1189 status = HCHK_STATUS_L7OKD;
1190 }
Simon Horman4a741432013-02-23 15:35:38 +09001191 } else if (!strncasecmp(check->bi->data, "down", strlen("down"))) {
Simon Hormana2b9dad2013-02-12 10:45:54 +09001192 down_cmd = "down";
Simon Horman4a741432013-02-23 15:35:38 +09001193 } else if (!strncasecmp(check->bi->data, "stopped", strlen("stopped"))) {
Simon Hormana2b9dad2013-02-12 10:45:54 +09001194 down_cmd = "stopped";
Simon Horman4a741432013-02-23 15:35:38 +09001195 } else if (!strncasecmp(check->bi->data, "fail", strlen("fail"))) {
Simon Hormana2b9dad2013-02-12 10:45:54 +09001196 down_cmd = "fail";
1197 }
1198
1199 if (down_cmd) {
Simon Horman4a741432013-02-23 15:35:38 +09001200 const char *end = check->bi->data + strlen(down_cmd);
Simon Hormana2b9dad2013-02-12 10:45:54 +09001201 /*
1202 * The command keyword must terminated the string or
1203 * be followed by a blank.
1204 */
Willy Tarreau8b4c3762013-02-13 12:47:12 +01001205 if (end[0] == '\0' || end[0] == ' ' || end[0] == '\t') {
Simon Hormana2b9dad2013-02-12 10:45:54 +09001206 status = HCHK_STATUS_L7STS;
Simon Horman80fefae2013-11-25 10:46:34 +09001207 desc = check->bi->data;
Simon Hormana2b9dad2013-02-12 10:45:54 +09001208 }
1209 }
1210
Simon Horman4a741432013-02-23 15:35:38 +09001211 set_server_check_status(check, status, desc);
Simon Horman8c3d0be2013-11-25 10:46:40 +09001212 set_server_drain_state(check->server);
Simon Hormana2b9dad2013-02-12 10:45:54 +09001213 break;
1214 }
1215
Willy Tarreau1620ec32011-08-06 17:05:02 +02001216 case PR_O2_PGSQL_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001217 if (!done && check->bi->i < 9)
Rauf Kuliyev38b41562011-01-04 15:14:13 +01001218 goto wait_more_data;
1219
Simon Horman4a741432013-02-23 15:35:38 +09001220 if (check->bi->data[0] == 'R') {
1221 set_server_check_status(check, HCHK_STATUS_L7OKD, "PostgreSQL server is ok");
Rauf Kuliyev38b41562011-01-04 15:14:13 +01001222 }
1223 else {
Simon Horman4a741432013-02-23 15:35:38 +09001224 if ((check->bi->data[0] == 'E') && (check->bi->data[5]!=0) && (check->bi->data[6]!=0))
1225 desc = &check->bi->data[6];
Rauf Kuliyev38b41562011-01-04 15:14:13 +01001226 else
1227 desc = "PostgreSQL unknown error";
1228
Simon Horman4a741432013-02-23 15:35:38 +09001229 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Rauf Kuliyev38b41562011-01-04 15:14:13 +01001230 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001231 break;
1232
1233 case PR_O2_REDIS_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001234 if (!done && check->bi->i < 7)
Hervé COMMOWICKec032d62011-08-05 16:23:48 +02001235 goto wait_more_data;
1236
Simon Horman4a741432013-02-23 15:35:38 +09001237 if (strcmp(check->bi->data, "+PONG\r\n") == 0) {
1238 set_server_check_status(check, HCHK_STATUS_L7OKD, "Redis server is ok");
Hervé COMMOWICKec032d62011-08-05 16:23:48 +02001239 }
1240 else {
Simon Horman4a741432013-02-23 15:35:38 +09001241 set_server_check_status(check, HCHK_STATUS_L7STS, check->bi->data);
Hervé COMMOWICKec032d62011-08-05 16:23:48 +02001242 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001243 break;
1244
1245 case PR_O2_MYSQL_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001246 if (!done && check->bi->i < 5)
Willy Tarreau03938182010-03-17 21:52:07 +01001247 goto wait_more_data;
1248
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001249 if (s->proxy->check_len == 0) { // old mode
Simon Horman4a741432013-02-23 15:35:38 +09001250 if (*(check->bi->data + 4) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001251 /* We set the MySQL Version in description for information purpose
1252 * FIXME : it can be cool to use MySQL Version for other purpose,
1253 * like mark as down old MySQL server.
1254 */
Simon Horman4a741432013-02-23 15:35:38 +09001255 if (check->bi->i > 51) {
1256 desc = ltrim(check->bi->data + 5, ' ');
1257 set_server_check_status(check, HCHK_STATUS_L7OKD, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001258 }
1259 else {
1260 if (!done)
1261 goto wait_more_data;
1262 /* it seems we have a OK packet but without a valid length,
1263 * it must be a protocol error
1264 */
Simon Horman4a741432013-02-23 15:35:38 +09001265 set_server_check_status(check, HCHK_STATUS_L7RSP, check->bi->data);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001266 }
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001267 }
1268 else {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001269 /* An error message is attached in the Error packet */
Simon Horman4a741432013-02-23 15:35:38 +09001270 desc = ltrim(check->bi->data + 7, ' ');
1271 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001272 }
1273 } else {
Simon Horman4a741432013-02-23 15:35:38 +09001274 unsigned int first_packet_len = ((unsigned int) *check->bi->data) +
1275 (((unsigned int) *(check->bi->data + 1)) << 8) +
1276 (((unsigned int) *(check->bi->data + 2)) << 16);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001277
Simon Horman4a741432013-02-23 15:35:38 +09001278 if (check->bi->i == first_packet_len + 4) {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001279 /* MySQL Error packet always begin with field_count = 0xff */
Simon Horman4a741432013-02-23 15:35:38 +09001280 if (*(check->bi->data + 4) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001281 /* We have only one MySQL packet and it is a Handshake Initialization packet
1282 * but we need to have a second packet to know if it is alright
1283 */
Simon Horman4a741432013-02-23 15:35:38 +09001284 if (!done && check->bi->i < first_packet_len + 5)
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001285 goto wait_more_data;
1286 }
1287 else {
1288 /* We have only one packet and it is an Error packet,
1289 * an error message is attached, so we can display it
1290 */
Simon Horman4a741432013-02-23 15:35:38 +09001291 desc = &check->bi->data[7];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001292 //Warning("onlyoneERR: %s\n", desc);
Simon Horman4a741432013-02-23 15:35:38 +09001293 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001294 }
Simon Horman4a741432013-02-23 15:35:38 +09001295 } else if (check->bi->i > first_packet_len + 4) {
1296 unsigned int second_packet_len = ((unsigned int) *(check->bi->data + first_packet_len + 4)) +
1297 (((unsigned int) *(check->bi->data + first_packet_len + 5)) << 8) +
1298 (((unsigned int) *(check->bi->data + first_packet_len + 6)) << 16);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001299
Simon Horman4a741432013-02-23 15:35:38 +09001300 if (check->bi->i == first_packet_len + 4 + second_packet_len + 4 ) {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001301 /* We have 2 packets and that's good */
1302 /* Check if the second packet is a MySQL Error packet or not */
Simon Horman4a741432013-02-23 15:35:38 +09001303 if (*(check->bi->data + first_packet_len + 8) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001304 /* No error packet */
1305 /* We set the MySQL Version in description for information purpose */
Simon Horman4a741432013-02-23 15:35:38 +09001306 desc = &check->bi->data[5];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001307 //Warning("2packetOK: %s\n", desc);
Simon Horman4a741432013-02-23 15:35:38 +09001308 set_server_check_status(check, HCHK_STATUS_L7OKD, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001309 }
1310 else {
1311 /* An error message is attached in the Error packet
1312 * so we can display it ! :)
1313 */
Simon Horman4a741432013-02-23 15:35:38 +09001314 desc = &check->bi->data[first_packet_len+11];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001315 //Warning("2packetERR: %s\n", desc);
Simon Horman4a741432013-02-23 15:35:38 +09001316 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001317 }
1318 }
1319 }
1320 else {
Willy Tarreau03938182010-03-17 21:52:07 +01001321 if (!done)
1322 goto wait_more_data;
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001323 /* it seems we have a Handshake Initialization packet but without a valid length,
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001324 * it must be a protocol error
1325 */
Simon Horman4a741432013-02-23 15:35:38 +09001326 desc = &check->bi->data[5];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001327 //Warning("protoerr: %s\n", desc);
Simon Horman4a741432013-02-23 15:35:38 +09001328 set_server_check_status(check, HCHK_STATUS_L7RSP, desc);
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001329 }
1330 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001331 break;
1332
1333 case PR_O2_LDAP_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001334 if (!done && check->bi->i < 14)
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001335 goto wait_more_data;
1336
1337 /* Check if the server speaks LDAP (ASN.1/BER)
1338 * http://en.wikipedia.org/wiki/Basic_Encoding_Rules
1339 * http://tools.ietf.org/html/rfc4511
1340 */
1341
1342 /* http://tools.ietf.org/html/rfc4511#section-4.1.1
1343 * LDAPMessage: 0x30: SEQUENCE
1344 */
Simon Horman4a741432013-02-23 15:35:38 +09001345 if ((check->bi->i < 14) || (*(check->bi->data) != '\x30')) {
1346 set_server_check_status(check, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001347 }
1348 else {
1349 /* size of LDAPMessage */
Simon Horman4a741432013-02-23 15:35:38 +09001350 msglen = (*(check->bi->data + 1) & 0x80) ? (*(check->bi->data + 1) & 0x7f) : 0;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001351
1352 /* http://tools.ietf.org/html/rfc4511#section-4.2.2
1353 * messageID: 0x02 0x01 0x01: INTEGER 1
1354 * protocolOp: 0x61: bindResponse
1355 */
1356 if ((msglen > 2) ||
Simon Horman4a741432013-02-23 15:35:38 +09001357 (memcmp(check->bi->data + 2 + msglen, "\x02\x01\x01\x61", 4) != 0)) {
1358 set_server_check_status(check, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001359
1360 goto out_wakeup;
1361 }
1362
1363 /* size of bindResponse */
Simon Horman4a741432013-02-23 15:35:38 +09001364 msglen += (*(check->bi->data + msglen + 6) & 0x80) ? (*(check->bi->data + msglen + 6) & 0x7f) : 0;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001365
1366 /* http://tools.ietf.org/html/rfc4511#section-4.1.9
1367 * ldapResult: 0x0a 0x01: ENUMERATION
1368 */
1369 if ((msglen > 4) ||
Simon Horman4a741432013-02-23 15:35:38 +09001370 (memcmp(check->bi->data + 7 + msglen, "\x0a\x01", 2) != 0)) {
1371 set_server_check_status(check, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001372
1373 goto out_wakeup;
1374 }
1375
1376 /* http://tools.ietf.org/html/rfc4511#section-4.1.9
1377 * resultCode
1378 */
Simon Horman4a741432013-02-23 15:35:38 +09001379 check->code = *(check->bi->data + msglen + 9);
1380 if (check->code) {
1381 set_server_check_status(check, HCHK_STATUS_L7STS, "See RFC: http://tools.ietf.org/html/rfc4511#section-4.1.9");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001382 } else {
Simon Horman4a741432013-02-23 15:35:38 +09001383 set_server_check_status(check, HCHK_STATUS_L7OKD, "Success");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001384 }
1385 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001386 break;
1387
1388 default:
Willy Tarreau06559ac2013-12-05 01:53:08 +01001389 /* for other checks (eg: pure TCP), delegate to the main task */
Willy Tarreau1620ec32011-08-06 17:05:02 +02001390 break;
1391 } /* switch */
Willy Tarreau83749182007-04-15 20:56:27 +02001392
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001393 out_wakeup:
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001394 /* collect possible new errors */
1395 if (conn->flags & CO_FL_ERROR)
1396 chk_report_conn_err(conn, 0, 0);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001397
Nick Chalk57b1bf72010-03-16 15:50:46 +00001398 /* Reset the check buffer... */
Simon Horman4a741432013-02-23 15:35:38 +09001399 *check->bi->data = '\0';
1400 check->bi->i = 0;
Nick Chalk57b1bf72010-03-16 15:50:46 +00001401
Willy Tarreaufd29cc52012-11-23 09:18:20 +01001402 /* Close the connection... We absolutely want to perform a hard close
1403 * and reset the connection if some data are pending, otherwise we end
1404 * up with many TIME_WAITs and eat all the source port range quickly.
1405 * To avoid sending RSTs all the time, we first try to drain pending
1406 * data.
1407 */
Willy Tarreauf1503172012-09-28 19:39:36 +02001408 if (conn->xprt && conn->xprt->shutw)
1409 conn->xprt->shutw(conn, 0);
Willy Tarreau2b57cb82013-06-10 19:56:38 +02001410
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001411 /* OK, let's not stay here forever */
Willy Tarreau6aaa1b82013-12-11 17:09:34 +01001412 if (check->result == CHK_RES_FAILED)
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001413 conn->flags |= CO_FL_ERROR;
1414
Willy Tarreaua522f802012-11-23 08:56:35 +01001415 __conn_data_stop_both(conn);
Willy Tarreaufdccded2008-08-29 18:19:04 +02001416 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreau3267d362012-08-17 23:53:56 +02001417 return;
Willy Tarreau03938182010-03-17 21:52:07 +01001418
1419 wait_more_data:
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001420 __conn_data_poll_recv(conn);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001421}
1422
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001423/*
1424 * This function is used only for server health-checks. It handles connection
1425 * status updates including errors. If necessary, it wakes the check task up.
1426 * It always returns 0.
1427 */
1428static int wake_srv_chk(struct connection *conn)
Willy Tarreau20bea422012-07-06 12:00:49 +02001429{
Simon Horman4a741432013-02-23 15:35:38 +09001430 struct check *check = conn->owner;
Willy Tarreau20bea422012-07-06 12:00:49 +02001431
Willy Tarreau6c560da2012-11-24 11:14:45 +01001432 if (unlikely(conn->flags & CO_FL_ERROR)) {
Willy Tarreau02b0f582013-12-03 15:42:33 +01001433 /* We may get error reports bypassing the I/O handlers, typically
1434 * the case when sending a pure TCP check which fails, then the I/O
1435 * handlers above are not called. This is completely handled by the
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001436 * main processing task so let's simply wake it up. If we get here,
1437 * we expect errno to still be valid.
1438 */
1439 chk_report_conn_err(conn, errno, 0);
1440
Willy Tarreau2d351b62013-12-05 02:36:25 +01001441 __conn_data_stop_both(conn);
1442 task_wakeup(check->task, TASK_WOKEN_IO);
1443 }
1444
Willy Tarreau6aaa1b82013-12-11 17:09:34 +01001445 if (check->result != CHK_RES_UNKNOWN) {
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001446 /* We're here because nobody wants to handle the error, so we
1447 * sure want to abort the hard way.
Willy Tarreau02b0f582013-12-03 15:42:33 +01001448 */
Willy Tarreauad38ace2013-12-15 14:19:38 +01001449 if (conn_ctrl_ready(conn) && !(conn->flags & CO_FL_SOCK_RD_SH)) {
1450 if (!(conn->flags & CO_FL_WAIT_RD) && conn->ctrl->drain && conn->ctrl->drain(conn->t.sock.fd))
1451 fdtab[conn->t.sock.fd].linger_risk = 0;
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001452 }
Willy Tarreauf79c8172013-10-21 16:30:56 +02001453 conn_force_close(conn);
Willy Tarreau2d351b62013-12-05 02:36:25 +01001454 }
Willy Tarreau3267d362012-08-17 23:53:56 +02001455 return 0;
Willy Tarreau20bea422012-07-06 12:00:49 +02001456}
1457
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001458struct data_cb check_conn_cb = {
1459 .recv = event_srv_chk_r,
1460 .send = event_srv_chk_w,
1461 .wake = wake_srv_chk,
1462};
1463
Willy Tarreaubaaee002006-06-26 02:48:02 +02001464/*
Willy Tarreau2e993902011-10-31 11:53:20 +01001465 * updates the server's weight during a warmup stage. Once the final weight is
1466 * reached, the task automatically stops. Note that any server status change
1467 * must have updated s->last_change accordingly.
1468 */
1469static struct task *server_warmup(struct task *t)
1470{
1471 struct server *s = t->context;
1472
1473 /* by default, plan on stopping the task */
1474 t->expire = TICK_ETERNITY;
1475 if ((s->state & (SRV_RUNNING|SRV_WARMINGUP|SRV_MAINTAIN)) != (SRV_RUNNING|SRV_WARMINGUP))
1476 return t;
1477
Willy Tarreau004e0452013-11-21 11:22:01 +01001478 server_recalc_eweight(s);
Willy Tarreau2e993902011-10-31 11:53:20 +01001479
1480 /* probably that we can refill this server with a bit more connections */
1481 check_for_pending(s);
1482
1483 /* get back there in 1 second or 1/20th of the slowstart interval,
1484 * whichever is greater, resulting in small 5% steps.
1485 */
1486 if (s->state & SRV_WARMINGUP)
1487 t->expire = tick_add(now_ms, MS_TO_TICKS(MAX(1000, s->slowstart / 20)));
1488 return t;
1489}
1490
1491/*
Willy Tarreaubaaee002006-06-26 02:48:02 +02001492 * manages a server health-check. Returns
1493 * the time the task accepts to wait, or TIME_ETERNITY for infinity.
1494 */
Simon Horman63a4a822012-03-19 07:24:41 +09001495static struct task *process_chk(struct task *t)
Willy Tarreaubaaee002006-06-26 02:48:02 +02001496{
Simon Horman4a741432013-02-23 15:35:38 +09001497 struct check *check = t->context;
1498 struct server *s = check->server;
1499 struct connection *conn = check->conn;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001500 int rv;
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001501 int ret;
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001502 int expired = tick_is_expired(t->expire, now_ms);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001503
Willy Tarreau2c115e52013-12-11 19:41:16 +01001504 if (!(check->state & CHK_ST_INPROGRESS)) {
Willy Tarreau5a78f362012-11-23 12:47:05 +01001505 /* no check currently running */
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001506 if (!expired) /* woke up too early */
Willy Tarreau26c25062009-03-08 09:38:41 +01001507 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001508
Simon Horman671b6f02013-11-25 10:46:39 +09001509 /* we don't send any health-checks when the proxy is
1510 * stopped, the server should not be checked or the check
1511 * is disabled.
Willy Tarreaubaaee002006-06-26 02:48:02 +02001512 */
Willy Tarreau0d924cc2013-12-11 21:26:24 +01001513 if (((check->state & (CHK_ST_ENABLED | CHK_ST_PAUSED)) != CHK_ST_ENABLED) ||
Willy Tarreau33a08db2013-12-11 21:03:31 +01001514 s->proxy->state == PR_STSTOPPED)
Willy Tarreau5a78f362012-11-23 12:47:05 +01001515 goto reschedule;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001516
1517 /* we'll initiate a new check */
Simon Horman4a741432013-02-23 15:35:38 +09001518 set_server_check_status(check, HCHK_STATUS_START, NULL);
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001519
Willy Tarreau2c115e52013-12-11 19:41:16 +01001520 check->state |= CHK_ST_INPROGRESS;
Simon Horman4a741432013-02-23 15:35:38 +09001521 check->bi->p = check->bi->data;
1522 check->bi->i = 0;
1523 check->bo->p = check->bo->data;
1524 check->bo->o = 0;
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001525
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02001526 /* tcpcheck send/expect initialisation */
1527 if (check->type == PR_O2_TCPCHK_CHK)
1528 check->current_step = NULL;
1529
1530 /* prepare the check buffer.
1531 * This should not be used if check is the secondary agent check
1532 * of a server as s->proxy->check_req will relate to the
1533 * configuration of the primary check. Similarly, tcp-check uses
1534 * its own strings.
1535 */
Willy Tarreau33434322013-12-11 21:15:19 +01001536 if (check->type && check->type != PR_O2_TCPCHK_CHK && !(check->state & CHK_ST_AGENT)) {
Simon Horman4a741432013-02-23 15:35:38 +09001537 bo_putblk(check->bo, s->proxy->check_req, s->proxy->check_len);
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001538
1539 /* we want to check if this host replies to HTTP or SSLv3 requests
1540 * so we'll send the request, and won't wake the checker up now.
1541 */
Simon Horman4a741432013-02-23 15:35:38 +09001542 if ((check->type) == PR_O2_SSL3_CHK) {
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001543 /* SSL requires that we put Unix time in the request */
1544 int gmt_time = htonl(date.tv_sec);
Simon Horman4a741432013-02-23 15:35:38 +09001545 memcpy(check->bo->data + 11, &gmt_time, 4);
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001546 }
Simon Horman4a741432013-02-23 15:35:38 +09001547 else if ((check->type) == PR_O2_HTTP_CHK) {
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001548 if (s->proxy->options2 & PR_O2_CHK_SNDST)
Simon Horman4a741432013-02-23 15:35:38 +09001549 bo_putblk(check->bo, trash.str, httpchk_build_status_header(s, trash.str));
1550 bo_putstr(check->bo, "\r\n");
1551 *check->bo->p = '\0'; /* to make gdb output easier to read */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001552 }
1553 }
1554
1555 /* prepare a new connection */
Willy Tarreau4bdae8a2013-10-14 17:29:15 +02001556 conn_init(conn);
Willy Tarreau910c6aa2013-10-24 15:08:37 +02001557 conn_prepare(conn, s->check_common.proto, s->check_common.xprt);
Willy Tarreau7abddb52013-10-24 15:31:04 +02001558 conn_attach(conn, check, &check_conn_cb);
Willy Tarreau4bdae8a2013-10-14 17:29:15 +02001559 conn->target = &s->obj_type;
Willy Tarreauc6f4ce82009-06-10 11:09:37 +02001560
Willy Tarreau5f2877a2012-10-26 19:57:58 +02001561 /* no client address */
1562 clear_addr(&conn->addr.from);
1563
Simon Horman66183002013-02-23 10:16:43 +09001564 if (is_addr(&s->check_common.addr))
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001565 /* we'll connect to the check addr specified on the server */
Simon Horman66183002013-02-23 10:16:43 +09001566 conn->addr.to = s->check_common.addr;
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001567 else
1568 /* we'll connect to the addr on the server */
1569 conn->addr.to = s->addr;
Willy Tarreauc6f4ce82009-06-10 11:09:37 +02001570
Simon Horman4a741432013-02-23 15:35:38 +09001571 set_host_port(&conn->addr.to, check->port);
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001572
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001573 /* It can return one of :
1574 * - SN_ERR_NONE if everything's OK
1575 * - SN_ERR_SRVTO if there are no more servers
1576 * - SN_ERR_SRVCL if the connection was refused by the server
1577 * - SN_ERR_PRXCOND if the connection has been limited by the proxy (maxconn)
1578 * - SN_ERR_RESOURCE if a system resource is lacking (eg: fd limits, ports, ...)
1579 * - SN_ERR_INTERNAL for any other purely internal errors
1580 * Additionnally, in the case of SN_ERR_RESOURCE, an emergency log will be emitted.
Willy Tarreau24db47e2012-11-23 14:16:39 +01001581 * Note that we try to prevent the network stack from sending the ACK during the
Willy Tarreauf0837b22012-11-24 10:24:27 +01001582 * connect() when a pure TCP check is used (without PROXY protocol).
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001583 */
Willy Tarreau8f46cca2013-03-04 20:07:44 +01001584 ret = SN_ERR_INTERNAL;
Simon Horman66183002013-02-23 10:16:43 +09001585 if (s->check_common.proto->connect)
Willy Tarreau57cd3e42013-10-24 22:01:26 +02001586 ret = s->check_common.proto->connect(conn, check->type, (check->type) ? 0 : 2);
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001587 conn->flags |= CO_FL_WAKE_DATA;
Willy Tarreau57cd3e42013-10-24 22:01:26 +02001588 if (s->check.send_proxy) {
1589 conn->send_proxy_ofs = 1;
1590 conn->flags |= CO_FL_SEND_PROXY;
1591 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001592
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001593 switch (ret) {
1594 case SN_ERR_NONE:
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001595 /* we allow up to min(inter, timeout.connect) for a connection
1596 * to establish but only when timeout.check is set
1597 * as it may be to short for a full check otherwise
1598 */
Simon Horman4a741432013-02-23 15:35:38 +09001599 t->expire = tick_add(now_ms, MS_TO_TICKS(check->inter));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001600
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001601 if (s->proxy->timeout.check && s->proxy->timeout.connect) {
1602 int t_con = tick_add(now_ms, s->proxy->timeout.connect);
1603 t->expire = tick_first(t->expire, t_con);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001604 }
Willy Tarreau06559ac2013-12-05 01:53:08 +01001605
1606 if (check->type)
1607 conn_data_want_recv(conn); /* prepare for reading a possible reply */
1608
Willy Tarreau5a78f362012-11-23 12:47:05 +01001609 goto reschedule;
1610
1611 case SN_ERR_SRVTO: /* ETIMEDOUT */
1612 case SN_ERR_SRVCL: /* ECONNREFUSED, ENETUNREACH, ... */
Simon Horman4a741432013-02-23 15:35:38 +09001613 set_server_check_status(check, HCHK_STATUS_L4CON, strerror(errno));
Willy Tarreau5a78f362012-11-23 12:47:05 +01001614 break;
1615 case SN_ERR_PRXCOND:
1616 case SN_ERR_RESOURCE:
1617 case SN_ERR_INTERNAL:
Simon Horman4a741432013-02-23 15:35:38 +09001618 set_server_check_status(check, HCHK_STATUS_SOCKERR, NULL);
Willy Tarreau5a78f362012-11-23 12:47:05 +01001619 break;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001620 }
1621
Willy Tarreau5a78f362012-11-23 12:47:05 +01001622 /* here, we have seen a synchronous error, no fd was allocated */
Willy Tarreau6b0a8502012-11-23 08:51:32 +01001623
Willy Tarreau2c115e52013-12-11 19:41:16 +01001624 check->state &= ~CHK_ST_INPROGRESS;
Simon Hormand8583062013-11-25 10:46:33 +09001625 check_failed(check);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001626
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001627 /* we allow up to min(inter, timeout.connect) for a connection
1628 * to establish but only when timeout.check is set
1629 * as it may be to short for a full check otherwise
1630 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001631 while (tick_is_expired(t->expire, now_ms)) {
1632 int t_con;
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001633
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001634 t_con = tick_add(t->expire, s->proxy->timeout.connect);
Simon Horman4a741432013-02-23 15:35:38 +09001635 t->expire = tick_add(t->expire, MS_TO_TICKS(check->inter));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001636
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001637 if (s->proxy->timeout.check)
1638 t->expire = tick_first(t->expire, t_con);
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001639 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001640 }
1641 else {
Willy Tarreauf1503172012-09-28 19:39:36 +02001642 /* there was a test running.
1643 * First, let's check whether there was an uncaught error,
1644 * which can happen on connect timeout or error.
1645 */
Willy Tarreau6aaa1b82013-12-11 17:09:34 +01001646 if (s->check.result == CHK_RES_UNKNOWN) {
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001647 /* good connection is enough for pure TCP check */
1648 if ((conn->flags & CO_FL_CONNECTED) && !check->type) {
Simon Horman4a741432013-02-23 15:35:38 +09001649 if (check->use_ssl)
1650 set_server_check_status(check, HCHK_STATUS_L6OK, NULL);
Willy Tarreauf1503172012-09-28 19:39:36 +02001651 else
Simon Horman4a741432013-02-23 15:35:38 +09001652 set_server_check_status(check, HCHK_STATUS_L4OK, NULL);
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001653 }
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001654 else if ((conn->flags & CO_FL_ERROR) || expired) {
1655 chk_report_conn_err(conn, 0, expired);
Willy Tarreauf1503172012-09-28 19:39:36 +02001656 }
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001657 else
1658 goto out_wait; /* timeout not reached, wait again */
Willy Tarreauf1503172012-09-28 19:39:36 +02001659 }
1660
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001661 /* check complete or aborted */
Willy Tarreau5ba04f62013-02-12 15:23:12 +01001662 if (conn->xprt) {
1663 /* The check was aborted and the connection was not yet closed.
1664 * This can happen upon timeout, or when an external event such
1665 * as a failed response coupled with "observe layer7" caused the
1666 * server state to be suddenly changed.
1667 */
Willy Tarreauad38ace2013-12-15 14:19:38 +01001668 if (conn_ctrl_ready(conn) && !(conn->flags & CO_FL_SOCK_RD_SH)) {
1669 if (!(conn->flags & CO_FL_WAIT_RD) && conn->ctrl->drain && conn->ctrl->drain(conn->t.sock.fd))
1670 fdtab[conn->t.sock.fd].linger_risk = 0;
Willy Tarreau2d351b62013-12-05 02:36:25 +01001671 }
Willy Tarreauf79c8172013-10-21 16:30:56 +02001672 conn_force_close(conn);
Willy Tarreau5ba04f62013-02-12 15:23:12 +01001673 }
1674
Willy Tarreau6aaa1b82013-12-11 17:09:34 +01001675 if (check->result == CHK_RES_FAILED) /* a failure or timeout detected */
Simon Hormand8583062013-11-25 10:46:33 +09001676 check_failed(check);
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001677 else { /* check was OK */
Willy Tarreau48494c02007-11-30 10:41:39 +01001678 /* we may have to add/remove this server from the LB group */
1679 if ((s->state & SRV_RUNNING) && (s->proxy->options & PR_O_DISABLE404)) {
Willy Tarreau6aaa1b82013-12-11 17:09:34 +01001680 if ((s->state & SRV_GOINGDOWN) && (check->result != CHK_RES_CONDPASS))
Simon Horman4a741432013-02-23 15:35:38 +09001681 set_server_enabled(check);
Willy Tarreau6aaa1b82013-12-11 17:09:34 +01001682 else if (!(s->state & SRV_GOINGDOWN) && (check->result == CHK_RES_CONDPASS))
Simon Horman4a741432013-02-23 15:35:38 +09001683 set_server_disabled(check);
Willy Tarreau48494c02007-11-30 10:41:39 +01001684 }
1685
Willy Tarreau9f708ab2013-12-23 14:04:17 +01001686 if (!(s->state & SRV_MAINTAIN) &&
1687 check->health < check->rise + check->fall - 1) {
Simon Horman125d0992013-02-24 17:23:38 +09001688 check->health++; /* was bad, stays for a while */
Simon Horman4a741432013-02-23 15:35:38 +09001689 set_server_up(check);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001690 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001691 }
Willy Tarreau2c115e52013-12-11 19:41:16 +01001692 check->state &= ~CHK_ST_INPROGRESS;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001693
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001694 rv = 0;
1695 if (global.spread_checks > 0) {
Simon Horman4a741432013-02-23 15:35:38 +09001696 rv = srv_getinter(check) * global.spread_checks / 100;
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001697 rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0)));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001698 }
Simon Horman4a741432013-02-23 15:35:38 +09001699 t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(check) + rv));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001700 }
Willy Tarreau5a78f362012-11-23 12:47:05 +01001701
1702 reschedule:
1703 while (tick_is_expired(t->expire, now_ms))
Simon Horman4a741432013-02-23 15:35:38 +09001704 t->expire = tick_add(t->expire, MS_TO_TICKS(check->inter));
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001705 out_wait:
Willy Tarreau26c25062009-03-08 09:38:41 +01001706 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001707}
1708
Simon Horman5c942422013-11-25 10:46:32 +09001709static int start_check_task(struct check *check, int mininter,
1710 int nbcheck, int srvpos)
1711{
1712 struct task *t;
1713 /* task for the check */
1714 if ((t = task_new()) == NULL) {
1715 Alert("Starting [%s:%s] check: out of memory.\n",
1716 check->server->proxy->id, check->server->id);
1717 return 0;
1718 }
1719
1720 check->task = t;
1721 t->process = process_chk;
1722 t->context = check;
1723
1724 /* check this every ms */
1725 t->expire = tick_add(now_ms,
1726 MS_TO_TICKS(((mininter &&
1727 mininter >= srv_getinter(check)) ?
1728 mininter : srv_getinter(check)) * srvpos / nbcheck));
1729 check->start = now;
1730 task_queue(t);
1731
1732 return 1;
1733}
1734
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001735/*
1736 * Start health-check.
1737 * Returns 0 if OK, -1 if error, and prints the error in this case.
1738 */
1739int start_checks() {
1740
1741 struct proxy *px;
1742 struct server *s;
1743 struct task *t;
Simon Horman4a741432013-02-23 15:35:38 +09001744 int nbcheck=0, mininter=0, srvpos=0;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001745
Willy Tarreau2c43a1e2007-10-14 23:05:39 +02001746 /* 1- count the checkers to run simultaneously.
1747 * We also determine the minimum interval among all of those which
1748 * have an interval larger than SRV_CHK_INTER_THRES. This interval
1749 * will be used to spread their start-up date. Those which have
Jamie Gloudon801a0a32012-08-25 00:18:33 -04001750 * a shorter interval will start independently and will not dictate
Willy Tarreau2c43a1e2007-10-14 23:05:39 +02001751 * too short an interval for all others.
1752 */
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001753 for (px = proxy; px; px = px->next) {
1754 for (s = px->srv; s; s = s->next) {
Willy Tarreaue7b73482013-11-21 11:50:50 +01001755 if (s->slowstart) {
1756 if ((t = task_new()) == NULL) {
1757 Alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
1758 return -1;
1759 }
1760 /* We need a warmup task that will be called when the server
1761 * state switches from down to up.
1762 */
1763 s->warmup = t;
1764 t->process = server_warmup;
1765 t->context = s;
1766 t->expire = TICK_ETERNITY;
1767 }
1768
Willy Tarreaud8514a22013-12-11 21:10:14 +01001769 if (s->check.state & CHK_ST_CONFIGURED) {
1770 nbcheck++;
1771 if ((srv_getinter(&s->check) >= SRV_CHK_INTER_THRES) &&
1772 (!mininter || mininter > srv_getinter(&s->check)))
1773 mininter = srv_getinter(&s->check);
1774 }
Willy Tarreau15f39102013-12-11 20:41:18 +01001775
Willy Tarreaud8514a22013-12-11 21:10:14 +01001776 if (s->agent.state & CHK_ST_CONFIGURED) {
1777 nbcheck++;
1778 if ((srv_getinter(&s->agent) >= SRV_CHK_INTER_THRES) &&
1779 (!mininter || mininter > srv_getinter(&s->agent)))
1780 mininter = srv_getinter(&s->agent);
1781 }
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001782 }
1783 }
1784
Simon Horman4a741432013-02-23 15:35:38 +09001785 if (!nbcheck)
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001786 return 0;
1787
1788 srand((unsigned)time(NULL));
1789
1790 /*
1791 * 2- start them as far as possible from each others. For this, we will
1792 * start them after their interval set to the min interval divided by
1793 * the number of servers, weighted by the server's position in the list.
1794 */
1795 for (px = proxy; px; px = px->next) {
1796 for (s = px->srv; s; s = s->next) {
Simon Hormand60d6912013-11-25 10:46:36 +09001797 /* A task for the main check */
Willy Tarreauff5ae352013-12-11 20:36:34 +01001798 if (s->check.state & CHK_ST_CONFIGURED) {
Simon Hormand60d6912013-11-25 10:46:36 +09001799 if (!start_check_task(&s->check, mininter, nbcheck, srvpos))
1800 return -1;
1801 srvpos++;
1802 }
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001803
Simon Hormand60d6912013-11-25 10:46:36 +09001804 /* A task for a auxiliary agent check */
Willy Tarreauff5ae352013-12-11 20:36:34 +01001805 if (s->agent.state & CHK_ST_CONFIGURED) {
Simon Hormand60d6912013-11-25 10:46:36 +09001806 if (!start_check_task(&s->agent, mininter, nbcheck, srvpos)) {
1807 return -1;
1808 }
1809 srvpos++;
1810 }
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001811 }
1812 }
1813 return 0;
1814}
Willy Tarreaubaaee002006-06-26 02:48:02 +02001815
1816/*
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001817 * Perform content verification check on data in s->check.buffer buffer.
Willy Tarreaubd741542010-03-16 18:46:54 +01001818 * The buffer MUST be terminated by a null byte before calling this function.
1819 * Sets server status appropriately. The caller is responsible for ensuring
1820 * that the buffer contains at least 13 characters. If <done> is zero, we may
1821 * return 0 to indicate that data is required to decide of a match.
1822 */
1823static int httpchk_expect(struct server *s, int done)
1824{
1825 static char status_msg[] = "HTTP status check returned code <000>";
1826 char status_code[] = "000";
1827 char *contentptr;
1828 int crlf;
1829 int ret;
1830
1831 switch (s->proxy->options2 & PR_O2_EXP_TYPE) {
1832 case PR_O2_EXP_STS:
1833 case PR_O2_EXP_RSTS:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001834 memcpy(status_code, s->check.bi->data + 9, 3);
1835 memcpy(status_msg + strlen(status_msg) - 4, s->check.bi->data + 9, 3);
Willy Tarreaubd741542010-03-16 18:46:54 +01001836
1837 if ((s->proxy->options2 & PR_O2_EXP_TYPE) == PR_O2_EXP_STS)
1838 ret = strncmp(s->proxy->expect_str, status_code, 3) == 0;
1839 else
1840 ret = regexec(s->proxy->expect_regex, status_code, MAX_MATCH, pmatch, 0) == 0;
1841
1842 /* we necessarily have the response, so there are no partial failures */
1843 if (s->proxy->options2 & PR_O2_EXP_INV)
1844 ret = !ret;
1845
Simon Horman4a741432013-02-23 15:35:38 +09001846 set_server_check_status(&s->check, ret ? HCHK_STATUS_L7OKD : HCHK_STATUS_L7STS, status_msg);
Willy Tarreaubd741542010-03-16 18:46:54 +01001847 break;
1848
1849 case PR_O2_EXP_STR:
1850 case PR_O2_EXP_RSTR:
1851 /* very simple response parser: ignore CR and only count consecutive LFs,
1852 * stop with contentptr pointing to first char after the double CRLF or
1853 * to '\0' if crlf < 2.
1854 */
1855 crlf = 0;
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001856 for (contentptr = s->check.bi->data; *contentptr; contentptr++) {
Willy Tarreaubd741542010-03-16 18:46:54 +01001857 if (crlf >= 2)
1858 break;
1859 if (*contentptr == '\r')
1860 continue;
1861 else if (*contentptr == '\n')
1862 crlf++;
1863 else
1864 crlf = 0;
1865 }
1866
1867 /* Check that response contains a body... */
1868 if (crlf < 2) {
1869 if (!done)
1870 return 0;
1871
Simon Horman4a741432013-02-23 15:35:38 +09001872 set_server_check_status(&s->check, HCHK_STATUS_L7RSP,
Willy Tarreaubd741542010-03-16 18:46:54 +01001873 "HTTP content check could not find a response body");
1874 return 1;
1875 }
1876
1877 /* Check that response body is not empty... */
1878 if (*contentptr == '\0') {
Willy Tarreaua164fb52011-04-13 09:32:41 +02001879 if (!done)
1880 return 0;
1881
Simon Horman4a741432013-02-23 15:35:38 +09001882 set_server_check_status(&s->check, HCHK_STATUS_L7RSP,
Willy Tarreaubd741542010-03-16 18:46:54 +01001883 "HTTP content check found empty response body");
1884 return 1;
1885 }
1886
1887 /* Check the response content against the supplied string
1888 * or regex... */
1889 if ((s->proxy->options2 & PR_O2_EXP_TYPE) == PR_O2_EXP_STR)
1890 ret = strstr(contentptr, s->proxy->expect_str) != NULL;
1891 else
1892 ret = regexec(s->proxy->expect_regex, contentptr, MAX_MATCH, pmatch, 0) == 0;
1893
1894 /* if we don't match, we may need to wait more */
1895 if (!ret && !done)
1896 return 0;
1897
1898 if (ret) {
1899 /* content matched */
1900 if (s->proxy->options2 & PR_O2_EXP_INV)
Simon Horman4a741432013-02-23 15:35:38 +09001901 set_server_check_status(&s->check, HCHK_STATUS_L7RSP,
Willy Tarreaubd741542010-03-16 18:46:54 +01001902 "HTTP check matched unwanted content");
1903 else
Simon Horman4a741432013-02-23 15:35:38 +09001904 set_server_check_status(&s->check, HCHK_STATUS_L7OKD,
Willy Tarreaubd741542010-03-16 18:46:54 +01001905 "HTTP content check matched");
1906 }
1907 else {
1908 if (s->proxy->options2 & PR_O2_EXP_INV)
Simon Horman4a741432013-02-23 15:35:38 +09001909 set_server_check_status(&s->check, HCHK_STATUS_L7OKD,
Willy Tarreaubd741542010-03-16 18:46:54 +01001910 "HTTP check did not match unwanted content");
1911 else
Simon Horman4a741432013-02-23 15:35:38 +09001912 set_server_check_status(&s->check, HCHK_STATUS_L7RSP,
Willy Tarreaubd741542010-03-16 18:46:54 +01001913 "HTTP content check did not match");
1914 }
1915 break;
1916 }
1917 return 1;
1918}
1919
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02001920/*
1921 * return the id of a step in a send/expect session
1922 */
1923static int tcpcheck_get_step_id(struct server *s)
1924{
1925 struct tcpcheck_rule *cur = NULL, *next = NULL;
1926 int i = 0;
1927
1928 cur = s->check.current_step;
1929
1930 /* no step => first step */
1931 if (cur == NULL)
1932 return 1;
1933
1934 /* increment i until current step */
1935 list_for_each_entry(next, &s->proxy->tcpcheck_rules, list) {
1936 if (next->list.p == &cur->list)
1937 break;
1938 ++i;
1939 }
1940
1941 return i;
1942}
1943
1944static void tcpcheck_main(struct connection *conn)
1945{
1946 char *contentptr;
1947 unsigned int contentlen;
1948 struct list *head = NULL;
1949 struct tcpcheck_rule *cur = NULL;
1950 int done = 0, ret = 0;
1951
1952 struct check *check = conn->owner;
1953 struct server *s = check->server;
1954 struct task *t = check->task;
1955
1956 /* don't do anything until the connection is established */
1957 if (!(conn->flags & CO_FL_CONNECTED)) {
1958 /* update expire time, should be done by process_chk */
1959 /* we allow up to min(inter, timeout.connect) for a connection
1960 * to establish but only when timeout.check is set
1961 * as it may be to short for a full check otherwise
1962 */
1963 while (tick_is_expired(t->expire, now_ms)) {
1964 int t_con;
1965
1966 t_con = tick_add(t->expire, s->proxy->timeout.connect);
1967 t->expire = tick_add(t->expire, MS_TO_TICKS(check->inter));
1968
1969 if (s->proxy->timeout.check)
1970 t->expire = tick_first(t->expire, t_con);
1971 }
1972 return;
1973 }
1974
1975 /* here, we know that the connection is established */
Willy Tarreau6aaa1b82013-12-11 17:09:34 +01001976 if (check->result != CHK_RES_UNKNOWN)
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02001977 goto out_end_tcpcheck;
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02001978
1979 /* head is be the first element of the double chained list */
1980 head = &s->proxy->tcpcheck_rules;
1981
1982 /* no step means first step
1983 * initialisation */
1984 if (check->current_step == NULL) {
1985 check->bo->p = check->bo->data;
1986 check->bo->o = 0;
1987 check->bi->p = check->bi->data;
1988 check->bi->i = 0;
1989 cur = check->current_step = LIST_ELEM(head->n, struct tcpcheck_rule *, list);
1990 t->expire = tick_add(now_ms, MS_TO_TICKS(check->inter));
1991 if (s->proxy->timeout.check)
1992 t->expire = tick_add_ifset(now_ms, s->proxy->timeout.check);
1993 }
1994 /* keep on processing step */
1995 else {
1996 cur = check->current_step;
1997 }
1998
Willy Tarreaufbe0edf2013-12-06 16:54:31 +01001999 if (conn->flags & CO_FL_HANDSHAKE)
2000 return;
2001
2002 /* It's only the rules which will enable send/recv */
2003 __conn_data_stop_both(conn);
2004
Willy Tarreauabca5b62013-12-06 14:19:25 +01002005 while (1) {
Willy Tarreaufbe0edf2013-12-06 16:54:31 +01002006 /* we have to try to flush the output buffer before reading, at the end,
2007 * or if we're about to send a string that does not fit in the remaining space.
2008 */
2009 if (check->bo->o &&
2010 (&cur->list == head ||
2011 check->current_step->action != TCPCHK_ACT_SEND ||
2012 check->current_step->string_len >= buffer_total_space(check->bo))) {
2013
2014 if ((conn->flags & CO_FL_WAIT_WR) ||
2015 conn->xprt->snd_buf(conn, check->bo, MSG_DONTWAIT | MSG_NOSIGNAL) <= 0) {
2016 if (conn->flags & CO_FL_ERROR) {
2017 chk_report_conn_err(conn, errno, 0);
2018 __conn_data_stop_both(conn);
2019 goto out_end_tcpcheck;
2020 }
2021 goto out_need_io;
Willy Tarreauabca5b62013-12-06 14:19:25 +01002022 }
Willy Tarreauabca5b62013-12-06 14:19:25 +01002023 }
2024
Willy Tarreaufbe0edf2013-12-06 16:54:31 +01002025 /* did we reach the end ? If so, let's check that everything was sent */
2026 if (&cur->list == head) {
2027 if (check->bo->o)
2028 goto out_need_io;
Willy Tarreauabca5b62013-12-06 14:19:25 +01002029 break;
Willy Tarreaufbe0edf2013-12-06 16:54:31 +01002030 }
Willy Tarreauabca5b62013-12-06 14:19:25 +01002031
Willy Tarreau98aec9f2013-12-06 16:16:41 +01002032 if (check->current_step->action == TCPCHK_ACT_SEND) {
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002033 /* reset the read buffer */
2034 if (*check->bi->data != '\0') {
2035 *check->bi->data = '\0';
2036 check->bi->i = 0;
2037 }
2038
2039 if (conn->flags & (CO_FL_SOCK_WR_SH | CO_FL_DATA_WR_SH)) {
2040 conn->flags |= CO_FL_ERROR;
2041 chk_report_conn_err(conn, 0, 0);
2042 goto out_end_tcpcheck;
2043 }
2044
Willy Tarreauabca5b62013-12-06 14:19:25 +01002045 if (check->current_step->string_len >= check->bo->size) {
2046 chunk_printf(&trash, "tcp-check send : string too large (%d) for buffer size (%d) at step %d",
2047 check->current_step->string_len, check->bo->size,
2048 tcpcheck_get_step_id(s));
2049 set_server_check_status(check, HCHK_STATUS_L7RSP, trash.str);
2050 goto out_end_tcpcheck;
2051 }
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002052
Willy Tarreaufbe0edf2013-12-06 16:54:31 +01002053 /* do not try to send if there is no space */
2054 if (check->current_step->string_len >= buffer_total_space(check->bo))
2055 continue;
2056
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002057 bo_putblk(check->bo, check->current_step->string, check->current_step->string_len);
2058 *check->bo->p = '\0'; /* to make gdb output easier to read */
2059
Willy Tarreauabca5b62013-12-06 14:19:25 +01002060 /* go to next rule and try to send */
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002061 cur = (struct tcpcheck_rule *)cur->list.n;
2062 check->current_step = cur;
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002063 } /* end 'send' */
Willy Tarreau98aec9f2013-12-06 16:16:41 +01002064 else if (check->current_step->action == TCPCHK_ACT_EXPECT) {
Willy Tarreau6aaa1b82013-12-11 17:09:34 +01002065 if (unlikely(check->result == CHK_RES_FAILED))
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002066 goto out_end_tcpcheck;
2067
Willy Tarreaufbe0edf2013-12-06 16:54:31 +01002068 if ((conn->flags & CO_FL_WAIT_RD) ||
2069 conn->xprt->rcv_buf(conn, check->bi, buffer_total_space(check->bi)) <= 0) {
2070 if (conn->flags & (CO_FL_ERROR | CO_FL_SOCK_RD_SH | CO_FL_DATA_RD_SH)) {
2071 done = 1;
2072 if ((conn->flags & CO_FL_ERROR) && !check->bi->i) {
2073 /* Report network errors only if we got no other data. Otherwise
2074 * we'll let the upper layers decide whether the response is OK
2075 * or not. It is very common that an RST sent by the server is
2076 * reported as an error just after the last data chunk.
2077 */
2078 chk_report_conn_err(conn, errno, 0);
2079 goto out_end_tcpcheck;
2080 }
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002081 }
Willy Tarreaufbe0edf2013-12-06 16:54:31 +01002082 else
2083 goto out_need_io;
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002084 }
2085
2086 /* Intermediate or complete response received.
2087 * Terminate string in check->bi->data buffer.
2088 */
2089 if (check->bi->i < check->bi->size) {
2090 check->bi->data[check->bi->i] = '\0';
2091 }
2092 else {
2093 check->bi->data[check->bi->i - 1] = '\0';
2094 done = 1; /* buffer full, don't wait for more data */
2095 }
2096
2097 contentptr = check->bi->data;
2098 contentlen = check->bi->i;
2099
2100 /* Check that response body is not empty... */
2101 if (*contentptr == '\0') {
2102 if (!done)
Willy Tarreaufbe0edf2013-12-06 16:54:31 +01002103 continue;
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002104
2105 /* empty response */
2106 chunk_printf(&trash, "TCPCHK got an empty response at step %d",
2107 tcpcheck_get_step_id(s));
2108 set_server_check_status(check, HCHK_STATUS_L7RSP, trash.str);
2109
2110 goto out_end_tcpcheck;
2111 }
2112
2113 if (!done && (cur->string != NULL) && (check->bi->i < cur->string_len) )
Willy Tarreaua970c282013-12-06 12:47:19 +01002114 continue; /* try to read more */
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002115
Willy Tarreaua970c282013-12-06 12:47:19 +01002116 tcpcheck_expect:
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002117 if (cur->string != NULL)
2118 ret = my_memmem(contentptr, contentlen, cur->string, cur->string_len) != NULL;
2119 else if (cur->expect_regex != NULL)
2120 ret = regexec(cur->expect_regex, contentptr, MAX_MATCH, pmatch, 0) == 0;
2121
2122 if (!ret && !done)
Willy Tarreaua970c282013-12-06 12:47:19 +01002123 continue; /* try to read more */
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002124
2125 /* matched */
2126 if (ret) {
2127 /* matched but we did not want to => ERROR */
2128 if (cur->inverse) {
2129 /* we were looking for a string */
2130 if (cur->string != NULL) {
2131 chunk_printf(&trash, "TCPCHK matched unwanted content '%s' at step %d",
2132 cur->string, tcpcheck_get_step_id(s));
2133 }
2134 else {
2135 /* we were looking for a regex */
2136 chunk_printf(&trash, "TCPCHK matched unwanted content (regex) at step %d",
2137 tcpcheck_get_step_id(s));
2138 }
2139 set_server_check_status(check, HCHK_STATUS_L7RSP, trash.str);
2140 goto out_end_tcpcheck;
2141 }
2142 /* matched and was supposed to => OK, next step */
2143 else {
2144 cur = (struct tcpcheck_rule*)cur->list.n;
2145 check->current_step = cur;
Willy Tarreau98aec9f2013-12-06 16:16:41 +01002146 if (check->current_step->action == TCPCHK_ACT_EXPECT)
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002147 goto tcpcheck_expect;
2148 __conn_data_stop_recv(conn);
2149 }
2150 }
2151 else {
2152 /* not matched */
2153 /* not matched and was not supposed to => OK, next step */
2154 if (cur->inverse) {
2155 cur = (struct tcpcheck_rule*)cur->list.n;
2156 check->current_step = cur;
Willy Tarreau98aec9f2013-12-06 16:16:41 +01002157 if (check->current_step->action == TCPCHK_ACT_EXPECT)
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002158 goto tcpcheck_expect;
2159 __conn_data_stop_recv(conn);
2160 }
2161 /* not matched but was supposed to => ERROR */
2162 else {
2163 /* we were looking for a string */
2164 if (cur->string != NULL) {
2165 chunk_printf(&trash, "TCPCHK did not match content '%s' at step %d",
2166 cur->string, tcpcheck_get_step_id(s));
2167 }
2168 else {
2169 /* we were looking for a regex */
2170 chunk_printf(&trash, "TCPCHK did not match content (regex) at step %d",
2171 tcpcheck_get_step_id(s));
2172 }
2173 set_server_check_status(check, HCHK_STATUS_L7RSP, trash.str);
2174 goto out_end_tcpcheck;
2175 }
2176 }
2177 } /* end expect */
2178 } /* end loop over double chained step list */
2179
2180 set_server_check_status(check, HCHK_STATUS_L7OKD, "(tcp-check)");
2181 goto out_end_tcpcheck;
2182
Willy Tarreaufbe0edf2013-12-06 16:54:31 +01002183 out_need_io:
2184 if (check->bo->o)
2185 __conn_data_want_send(conn);
2186
2187 if (check->current_step->action == TCPCHK_ACT_EXPECT)
2188 __conn_data_want_recv(conn);
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002189 return;
2190
2191 out_end_tcpcheck:
2192 /* collect possible new errors */
2193 if (conn->flags & CO_FL_ERROR)
2194 chk_report_conn_err(conn, 0, 0);
2195
2196 /* Close the connection... We absolutely want to perform a hard close
2197 * and reset the connection if some data are pending, otherwise we end
2198 * up with many TIME_WAITs and eat all the source port range quickly.
2199 * To avoid sending RSTs all the time, we first try to drain pending
2200 * data.
2201 */
2202 if (conn->xprt && conn->xprt->shutw)
2203 conn->xprt->shutw(conn, 0);
2204
2205 check->current_step = NULL;
2206
Willy Tarreau6aaa1b82013-12-11 17:09:34 +01002207 if (check->result == CHK_RES_FAILED)
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +02002208 conn->flags |= CO_FL_ERROR;
2209
2210 __conn_data_stop_both(conn);
2211 task_wakeup(t, TASK_WOKEN_IO);
2212
2213 return;
2214}
2215
2216
Willy Tarreaubd741542010-03-16 18:46:54 +01002217/*
Willy Tarreaubaaee002006-06-26 02:48:02 +02002218 * Local variables:
2219 * c-indent-level: 8
2220 * c-basic-offset: 8
2221 * End:
2222 */