blob: 91ec4c958943aa37fcc3bdfb4b4065f6cefba0be [file] [log] [blame]
</
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
2 * Health-checks functions.
3 *
Willy Tarreau26c25062009-03-08 09:38:41 +01004 * Copyright 2000-2009 Willy Tarreau <w@1wt.eu>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02005 * Copyright 2007-2009 Krzysztof Piotr Oledzki <ole@ans.pl>
Willy Tarreaubaaee002006-06-26 02:48:02 +02006 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 *
12 */
13
Willy Tarreaub8816082008-01-18 12:18:15 +010014#include <assert.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020015#include <ctype.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020016#include <errno.h>
17#include <fcntl.h>
18#include <stdio.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020019#include <stdlib.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020020#include <string.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020021#include <time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020022#include <unistd.h>
natalie.chen7c92a9b2015-04-07 11:27:02 +080023#include <stdarg.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020024#include <sys/socket.h>
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +040025#include <sys/types.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020026#include <netinet/in.h>
Willy Tarreau1274bc42009-07-15 07:16:31 +020027#include <netinet/tcp.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020028#include <arpa/inet.h>
29
Willy Tarreauc7e42382012-08-24 19:22:53 +020030#include <common/chunk.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020031#include <common/compat.h>
32#include <common/config.h>
33#include <common/mini-clist.h>
Willy Tarreau83749182007-04-15 20:56:27 +020034#include <common/standard.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020035#include <common/time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020036
37#include <types/global.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020038
Baptiste Assmann69e273f2013-12-11 00:52:19 +010039#ifdef USE_OPENSSL
40#include <types/ssl_sock.h>
41#include <proto/ssl_sock.h>
42#endif /* USE_OPENSSL */
43
Willy Tarreaubaaee002006-06-26 02:48:02 +020044#include <proto/backend.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020045#include <proto/checks.h>
Simon Hormana2b9dad2013-02-12 10:45:54 +090046#include <proto/dumpstats.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020047#include <proto/fd.h>
48#include <proto/log.h>
49#include <proto/queue.h>
Willy Tarreauc6f4ce82009-06-10 11:09:37 +020050#include <proto/port_range.h>
Willy Tarreau3d300592007-03-18 18:34:41 +010051#include <proto/proto_http.h>
Willy Tarreaue8c66af2008-01-13 18:40:14 +010052#include <proto/proto_tcp.h>
Baptiste Assmann69e273f2013-12-11 00:52:19 +010053#include <proto/protocol.h>
Willy Tarreau2b5652f2006-12-31 17:46:05 +010054#include <proto/proxy.h>
Willy Tarreaufb56aab2012-09-28 14:40:02 +020055#include <proto/raw_sock.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020056#include <proto/server.h>
Simon Hormane0d1bfb2011-06-21 14:34:58 +090057#include <proto/session.h>
Willy Tarreau9e000c62011-03-10 14:03:36 +010058#include <proto/stream_interface.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020059#include <proto/task.h>
60
Willy Tarreaubd741542010-03-16 18:46:54 +010061static int httpchk_expect(struct server *s, int done);
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +020062static int tcpcheck_get_step_id(struct server *);
63static void tcpcheck_main(struct connection *);
Willy Tarreaubd741542010-03-16 18:46:54 +010064
Simon Horman63a4a822012-03-19 07:24:41 +090065static const struct check_status check_statuses[HCHK_STATUS_SIZE] = {
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010066 [HCHK_STATUS_UNKNOWN] = { CHK_RES_UNKNOWN, "UNK", "Unknown" },
67 [HCHK_STATUS_INI] = { CHK_RES_UNKNOWN, "INI", "Initializing" },
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020068 [HCHK_STATUS_START] = { /* SPECIAL STATUS*/ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020069
Willy Tarreau23964182014-05-20 20:56:30 +020070 /* Below we have finished checks */
71 [HCHK_STATUS_CHECKED] = { CHK_RES_NEUTRAL, "CHECKED", "No status change" },
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010072 [HCHK_STATUS_HANA] = { CHK_RES_FAILED, "HANA", "Health analyze" },
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010073
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010074 [HCHK_STATUS_SOCKERR] = { CHK_RES_FAILED, "SOCKERR", "Socket error" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020075
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010076 [HCHK_STATUS_L4OK] = { CHK_RES_PASSED, "L4OK", "Layer4 check passed" },
77 [HCHK_STATUS_L4TOUT] = { CHK_RES_FAILED, "L4TOUT", "Layer4 timeout" },
78 [HCHK_STATUS_L4CON] = { CHK_RES_FAILED, "L4CON", "Layer4 connection problem" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020079
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010080 [HCHK_STATUS_L6OK] = { CHK_RES_PASSED, "L6OK", "Layer6 check passed" },
81 [HCHK_STATUS_L6TOUT] = { CHK_RES_FAILED, "L6TOUT", "Layer6 timeout" },
82 [HCHK_STATUS_L6RSP] = { CHK_RES_FAILED, "L6RSP", "Layer6 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020083
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010084 [HCHK_STATUS_L7TOUT] = { CHK_RES_FAILED, "L7TOUT", "Layer7 timeout" },
85 [HCHK_STATUS_L7RSP] = { CHK_RES_FAILED, "L7RSP", "Layer7 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020086
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020087 [HCHK_STATUS_L57DATA] = { /* DUMMY STATUS */ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020088
Willy Tarreau6aaa1b82013-12-11 17:09:34 +010089 [HCHK_STATUS_L7OKD] = { CHK_RES_PASSED, "L7OK", "Layer7 check passed" },
90 [HCHK_STATUS_L7OKCD] = { CHK_RES_CONDPASS, "L7OKC", "Layer7 check conditionally passed" },
91 [HCHK_STATUS_L7STS] = { CHK_RES_FAILED, "L7STS", "Layer7 wrong status" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020092};
93
Simon Horman63a4a822012-03-19 07:24:41 +090094static const struct analyze_status analyze_statuses[HANA_STATUS_SIZE] = { /* 0: ignore, 1: error, 2: OK */
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010095 [HANA_STATUS_UNKNOWN] = { "Unknown", { 0, 0 }},
96
97 [HANA_STATUS_L4_OK] = { "L4 successful connection", { 2, 0 }},
98 [HANA_STATUS_L4_ERR] = { "L4 unsuccessful connection", { 1, 1 }},
99
100 [HANA_STATUS_HTTP_OK] = { "Correct http response", { 0, 2 }},
101 [HANA_STATUS_HTTP_STS] = { "Wrong http response", { 0, 1 }},
102 [HANA_STATUS_HTTP_HDRRSP] = { "Invalid http response (headers)", { 0, 1 }},
103 [HANA_STATUS_HTTP_RSP] = { "Invalid http response", { 0, 1 }},
104
105 [HANA_STATUS_HTTP_READ_ERROR] = { "Read error (http)", { 0, 1 }},
106 [HANA_STATUS_HTTP_READ_TIMEOUT] = { "Read timeout (http)", { 0, 1 }},
107 [HANA_STATUS_HTTP_BROKEN_PIPE] = { "Close from server (http)", { 0, 1 }},
108};
109
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200110/*
111 * Convert check_status code to description
112 */
113const char *get_check_status_description(short check_status) {
114
115 const char *desc;
116
117 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200118 desc = check_statuses[check_status].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200119 else
120 desc = NULL;
121
122 if (desc && *desc)
123 return desc;
124 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200125 return check_statuses[HCHK_STATUS_UNKNOWN].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200126}
127
128/*
129 * Convert check_status code to short info
130 */
131const char *get_check_status_info(short check_status) {
132
133 const char *info;
134
135 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200136 info = check_statuses[check_status].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200137 else
138 info = NULL;
139
140 if (info && *info)
141 return info;
142 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200143 return check_statuses[HCHK_STATUS_UNKNOWN].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200144}
145
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100146const char *get_analyze_status(short analyze_status) {
147
148 const char *desc;
149
150 if (analyze_status < HANA_STATUS_SIZE)
151 desc = analyze_statuses[analyze_status].desc;
152 else
153 desc = NULL;
154
155 if (desc && *desc)
156 return desc;
157 else
158 return analyze_statuses[HANA_STATUS_UNKNOWN].desc;
159}
160
Willy Tarreaua150cf12014-05-20 21:57:23 +0200161/* Builds a string containing some information about the health check's result.
162 * The output string is allocated from the trash chunks. If the check is NULL,
163 * NULL is returned. This is designed to be used when emitting logs about health
164 * checks.
Willy Tarreauddd329c2014-05-16 16:46:12 +0200165 */
Willy Tarreaua150cf12014-05-20 21:57:23 +0200166static const char *check_reason_string(struct check *check)
Willy Tarreauddd329c2014-05-16 16:46:12 +0200167{
Willy Tarreaua150cf12014-05-20 21:57:23 +0200168 struct chunk *msg;
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200169
Willy Tarreaua150cf12014-05-20 21:57:23 +0200170 if (!check)
171 return NULL;
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200172
Willy Tarreaua150cf12014-05-20 21:57:23 +0200173 msg = get_trash_chunk();
174 chunk_printf(msg, "reason: %s", get_check_status_description(check->status));
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200175
Willy Tarreaua150cf12014-05-20 21:57:23 +0200176 if (check->status >= HCHK_STATUS_L57DATA)
177 chunk_appendf(msg, ", code: %d", check->code);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200178
Willy Tarreaua150cf12014-05-20 21:57:23 +0200179 if (*check->desc) {
180 struct chunk src;
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200181
Willy Tarreaua150cf12014-05-20 21:57:23 +0200182 chunk_appendf(msg, ", info: \"");
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200183
Willy Tarreaua150cf12014-05-20 21:57:23 +0200184 chunk_initlen(&src, check->desc, 0, strlen(check->desc));
185 chunk_asciiencode(msg, &src, '"');
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200186
Willy Tarreaua150cf12014-05-20 21:57:23 +0200187 chunk_appendf(msg, "\"");
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200188 }
189
Willy Tarreaua150cf12014-05-20 21:57:23 +0200190 if (check->duration >= 0)
191 chunk_appendf(msg, ", check duration: %ldms", check->duration);
192
193 return msg->str;
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200194}
195
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200196/*
Simon Horman4a741432013-02-23 15:35:38 +0900197 * Set check->status, update check->duration and fill check->result with
Willy Tarreau7b1d47c2014-05-20 14:55:13 +0200198 * an adequate CHK_RES_* value. The new check->health is computed based
199 * on the result.
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200200 *
201 * Show information in logs about failed health check if server is UP
202 * or succeeded health checks if server is DOWN.
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200203 */
Simon Horman4a741432013-02-23 15:35:38 +0900204static void set_server_check_status(struct check *check, short status, const char *desc)
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100205{
Simon Horman4a741432013-02-23 15:35:38 +0900206 struct server *s = check->server;
Willy Tarreaubef1b322014-05-13 21:01:39 +0200207 short prev_status = check->status;
Willy Tarreau7b1d47c2014-05-20 14:55:13 +0200208 int report = 0;
Simon Horman4a741432013-02-23 15:35:38 +0900209
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200210 if (status == HCHK_STATUS_START) {
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100211 check->result = CHK_RES_UNKNOWN; /* no result yet */
Simon Horman4a741432013-02-23 15:35:38 +0900212 check->desc[0] = '\0';
213 check->start = now;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200214 return;
215 }
216
Simon Horman4a741432013-02-23 15:35:38 +0900217 if (!check->status)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200218 return;
219
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200220 if (desc && *desc) {
Simon Horman4a741432013-02-23 15:35:38 +0900221 strncpy(check->desc, desc, HCHK_DESC_LEN-1);
222 check->desc[HCHK_DESC_LEN-1] = '\0';
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200223 } else
Simon Horman4a741432013-02-23 15:35:38 +0900224 check->desc[0] = '\0';
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200225
Simon Horman4a741432013-02-23 15:35:38 +0900226 check->status = status;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200227 if (check_statuses[status].result)
Simon Horman4a741432013-02-23 15:35:38 +0900228 check->result = check_statuses[status].result;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200229
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100230 if (status == HCHK_STATUS_HANA)
Simon Horman4a741432013-02-23 15:35:38 +0900231 check->duration = -1;
232 else if (!tv_iszero(&check->start)) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200233 /* set_server_check_status() may be called more than once */
Simon Horman4a741432013-02-23 15:35:38 +0900234 check->duration = tv_ms_elapsed(&check->start, &now);
235 tv_zero(&check->start);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200236 }
237
Willy Tarreau23964182014-05-20 20:56:30 +0200238 /* no change is expected if no state change occurred */
239 if (check->result == CHK_RES_NEUTRAL)
240 return;
241
Willy Tarreau7b1d47c2014-05-20 14:55:13 +0200242 report = 0;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200243
Willy Tarreau7b1d47c2014-05-20 14:55:13 +0200244 switch (check->result) {
245 case CHK_RES_FAILED:
Willy Tarreau12634e12014-05-23 11:32:36 +0200246 /* Failure to connect to the agent as a secondary check should not
247 * cause the server to be marked down.
248 */
249 if ((!(check->state & CHK_ST_AGENT) ||
Simon Hormanbfb8f882015-02-26 11:26:17 +0900250 (check->status >= HCHK_STATUS_L57DATA)) &&
Willy Tarreau12634e12014-05-23 11:32:36 +0200251 (check->health >= check->rise)) {
Willy Tarreau7b1d47c2014-05-20 14:55:13 +0200252 s->counters.failed_checks++;
253 report = 1;
254 check->health--;
255 if (check->health < check->rise)
256 check->health = 0;
257 }
258 break;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200259
Willy Tarreau7b1d47c2014-05-20 14:55:13 +0200260 case CHK_RES_PASSED:
261 case CHK_RES_CONDPASS: /* "condpass" cannot make the first step but it OK after a "passed" */
262 if ((check->health < check->rise + check->fall - 1) &&
263 (check->result == CHK_RES_PASSED || check->health > 0)) {
264 report = 1;
265 check->health++;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200266
Willy Tarreau7b1d47c2014-05-20 14:55:13 +0200267 if (check->health >= check->rise)
268 check->health = check->rise + check->fall - 1; /* OK now */
269 }
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200270
Willy Tarreau7b1d47c2014-05-20 14:55:13 +0200271 /* clear consecutive_errors if observing is enabled */
272 if (s->onerror)
273 s->consecutive_errors = 0;
274 break;
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100275
Willy Tarreau7b1d47c2014-05-20 14:55:13 +0200276 default:
277 break;
278 }
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200279
Willy Tarreau7b1d47c2014-05-20 14:55:13 +0200280 if (s->proxy->options2 & PR_O2_LOGHCHKS &&
281 (status != prev_status || report)) {
282 chunk_printf(&trash,
Willy Tarreau12634e12014-05-23 11:32:36 +0200283 "%s check for %sserver %s/%s %s%s",
284 (check->state & CHK_ST_AGENT) ? "Agent" : "Health",
Willy Tarreauc93cd162014-05-13 15:54:22 +0200285 s->flags & SRV_F_BACKUP ? "backup " : "",
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100286 s->proxy->id, s->id,
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100287 (check->result == CHK_RES_CONDPASS) ? "conditionally ":"",
Willy Tarreau7b1d47c2014-05-20 14:55:13 +0200288 (check->result >= CHK_RES_PASSED) ? "succeeded" : "failed");
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200289
Willy Tarreaua150cf12014-05-20 21:57:23 +0200290 srv_append_status(&trash, s, check_reason_string(check), -1, 0);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200291
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100292 chunk_appendf(&trash, ", status: %d/%d %s",
Willy Tarreau7b1d47c2014-05-20 14:55:13 +0200293 (check->health >= check->rise) ? check->health - check->rise + 1 : check->health,
294 (check->health >= check->rise) ? check->fall : check->rise,
295 (check->health >= check->rise) ? (s->uweight ? "UP" : "DRAIN") : "DOWN");
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200296
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100297 Warning("%s.\n", trash.str);
natalie.chen2d0e45b2015-04-22 14:10:12 +0800298
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100299 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200300 }
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200301}
302
Willy Tarreau4eec5472014-05-20 22:32:27 +0200303/* Marks the check <check>'s server down if the current check is already failed
304 * and the server is not down yet nor in maintenance.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200305 */
Willy Tarreau4eec5472014-05-20 22:32:27 +0200306static void check_notify_failure(struct check *check)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200307{
Simon Horman4a741432013-02-23 15:35:38 +0900308 struct server *s = check->server;
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900309
Willy Tarreau7b1d47c2014-05-20 14:55:13 +0200310 /* The agent secondary check should only cause a server to be marked
311 * as down if check->status is HCHK_STATUS_L7STS, which indicates
312 * that the agent returned "fail", "stopped" or "down".
313 * The implication here is that failure to connect to the agent
314 * as a secondary check should not cause the server to be marked
315 * down. */
316 if ((check->state & CHK_ST_AGENT) && check->status != HCHK_STATUS_L7STS)
317 return;
318
Willy Tarreau4eec5472014-05-20 22:32:27 +0200319 if (check->health > 0)
320 return;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100321
Willy Tarreau4eec5472014-05-20 22:32:27 +0200322 /* We only report a reason for the check if we did not do so previously */
323 srv_set_stopped(s, (!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? check_reason_string(check) : NULL);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200324}
325
Willy Tarreauaf549582014-05-16 17:37:50 +0200326/* Marks the check <check> as valid and tries to set its server up, provided
Willy Tarreau3e048382014-05-21 10:30:54 +0200327 * it isn't in maintenance, it is not tracking a down server and other checks
328 * comply. The rule is simple : by default, a server is up, unless any of the
329 * following conditions is true :
330 * - health check failed (check->health < rise)
331 * - agent check failed (agent->health < rise)
332 * - the server tracks a down server (track && track->state == STOPPED)
333 * Note that if the server has a slowstart, it will switch to STARTING instead
334 * of RUNNING. Also, only the health checks support the nolb mode, so the
335 * agent's success may not take the server out of this mode.
Willy Tarreauaf549582014-05-16 17:37:50 +0200336 */
Willy Tarreau3e048382014-05-21 10:30:54 +0200337static void check_notify_success(struct check *check)
Willy Tarreauaf549582014-05-16 17:37:50 +0200338{
Simon Horman4a741432013-02-23 15:35:38 +0900339 struct server *s = check->server;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100340
Willy Tarreauaf549582014-05-16 17:37:50 +0200341 if (s->admin & SRV_ADMF_MAINT)
342 return;
Cyril Bontécd19e512010-01-31 22:34:03 +0100343
Willy Tarreau3e048382014-05-21 10:30:54 +0200344 if (s->track && s->track->state == SRV_ST_STOPPED)
Willy Tarreauaf549582014-05-16 17:37:50 +0200345 return;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100346
Willy Tarreau3e048382014-05-21 10:30:54 +0200347 if ((s->check.state & CHK_ST_ENABLED) && (s->check.health < s->check.rise))
348 return;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100349
Willy Tarreau3e048382014-05-21 10:30:54 +0200350 if ((s->agent.state & CHK_ST_ENABLED) && (s->agent.health < s->agent.rise))
351 return;
Willy Tarreauaf549582014-05-16 17:37:50 +0200352
Willy Tarreau3e048382014-05-21 10:30:54 +0200353 if ((check->state & CHK_ST_AGENT) && s->state == SRV_ST_STOPPING)
354 return;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100355
Willy Tarreau3e048382014-05-21 10:30:54 +0200356 srv_set_running(s, (!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? check_reason_string(check) : NULL);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100357}
358
Willy Tarreaudb58b792014-05-21 13:57:23 +0200359/* Marks the check <check> as valid and tries to set its server into stopping mode
360 * if it was running or starting, and provided it isn't in maintenance and other
361 * checks comply. The conditions for the server to be marked in stopping mode are
362 * the same as for it to be turned up. Also, only the health checks support the
363 * nolb mode.
Willy Tarreauaf549582014-05-16 17:37:50 +0200364 */
Willy Tarreaudb58b792014-05-21 13:57:23 +0200365static void check_notify_stopping(struct check *check)
Willy Tarreauaf549582014-05-16 17:37:50 +0200366{
Simon Horman4a741432013-02-23 15:35:38 +0900367 struct server *s = check->server;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100368
Willy Tarreauaf549582014-05-16 17:37:50 +0200369 if (s->admin & SRV_ADMF_MAINT)
370 return;
371
Willy Tarreaudb58b792014-05-21 13:57:23 +0200372 if (check->state & CHK_ST_AGENT)
373 return;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100374
Willy Tarreaudb58b792014-05-21 13:57:23 +0200375 if (s->track && s->track->state == SRV_ST_STOPPED)
376 return;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100377
Willy Tarreaudb58b792014-05-21 13:57:23 +0200378 if ((s->check.state & CHK_ST_ENABLED) && (s->check.health < s->check.rise))
379 return;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100380
Willy Tarreaudb58b792014-05-21 13:57:23 +0200381 if ((s->agent.state & CHK_ST_ENABLED) && (s->agent.health < s->agent.rise))
382 return;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100383
Willy Tarreaudb58b792014-05-21 13:57:23 +0200384 srv_set_stopping(s, (!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? check_reason_string(check) : NULL);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100385}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200386
Willy Tarreau9fe7aae2013-12-31 23:47:37 +0100387/* note: use health_adjust() only, which first checks that the observe mode is
388 * enabled.
389 */
390void __health_adjust(struct server *s, short status)
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100391{
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100392 int failed;
393 int expire;
394
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100395 if (s->observe >= HANA_OBS_SIZE)
396 return;
397
Willy Tarreaubb956662013-01-24 00:37:39 +0100398 if (status >= HANA_STATUS_SIZE || !analyze_statuses[status].desc)
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100399 return;
400
401 switch (analyze_statuses[status].lr[s->observe - 1]) {
402 case 1:
403 failed = 1;
404 break;
405
406 case 2:
407 failed = 0;
408 break;
409
410 default:
411 return;
412 }
413
414 if (!failed) {
415 /* good: clear consecutive_errors */
416 s->consecutive_errors = 0;
417 return;
418 }
419
420 s->consecutive_errors++;
421
422 if (s->consecutive_errors < s->consecutive_errors_limit)
423 return;
424
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100425 chunk_printf(&trash, "Detected %d consecutive errors, last one was: %s",
426 s->consecutive_errors, get_analyze_status(status));
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100427
428 switch (s->onerror) {
429 case HANA_ONERR_FASTINTER:
430 /* force fastinter - nothing to do here as all modes force it */
431 break;
432
433 case HANA_ONERR_SUDDTH:
434 /* simulate a pre-fatal failed health check */
Simon Horman58c32972013-11-25 10:46:38 +0900435 if (s->check.health > s->check.rise)
436 s->check.health = s->check.rise + 1;
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100437
438 /* no break - fall through */
439
440 case HANA_ONERR_FAILCHK:
441 /* simulate a failed health check */
Simon Horman4a741432013-02-23 15:35:38 +0900442 set_server_check_status(&s->check, HCHK_STATUS_HANA, trash.str);
Willy Tarreau4eec5472014-05-20 22:32:27 +0200443 check_notify_failure(&s->check);
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100444 break;
445
446 case HANA_ONERR_MARKDWN:
447 /* mark server down */
Simon Horman58c32972013-11-25 10:46:38 +0900448 s->check.health = s->check.rise;
Simon Horman4a741432013-02-23 15:35:38 +0900449 set_server_check_status(&s->check, HCHK_STATUS_HANA, trash.str);
Willy Tarreau4eec5472014-05-20 22:32:27 +0200450 check_notify_failure(&s->check);
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100451 break;
452
453 default:
454 /* write a warning? */
455 break;
456 }
457
458 s->consecutive_errors = 0;
459 s->counters.failed_hana++;
460
Simon Horman66183002013-02-23 10:16:43 +0900461 if (s->check.fastinter) {
462 expire = tick_add(now_ms, MS_TO_TICKS(s->check.fastinter));
Sergiy Prykhodko1d57e502013-09-21 12:05:00 +0300463 if (s->check.task->expire > expire) {
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200464 s->check.task->expire = expire;
Sergiy Prykhodko1d57e502013-09-21 12:05:00 +0300465 /* requeue check task with new expire */
466 task_queue(s->check.task);
467 }
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100468 }
469}
470
Willy Tarreaua1dab552014-04-14 15:04:54 +0200471static int httpchk_build_status_header(struct server *s, char *buffer, int size)
Willy Tarreauef781042010-01-27 11:53:01 +0100472{
473 int sv_state;
474 int ratio;
475 int hlen = 0;
476 const char *srv_hlt_st[7] = { "DOWN", "DOWN %d/%d",
477 "UP %d/%d", "UP",
478 "NOLB %d/%d", "NOLB",
479 "no check" };
480
481 memcpy(buffer + hlen, "X-Haproxy-Server-State: ", 24);
482 hlen += 24;
483
Willy Tarreauff5ae352013-12-11 20:36:34 +0100484 if (!(s->check.state & CHK_ST_ENABLED))
485 sv_state = 6;
Willy Tarreau892337c2014-05-13 23:41:20 +0200486 else if (s->state != SRV_ST_STOPPED) {
Simon Horman58c32972013-11-25 10:46:38 +0900487 if (s->check.health == s->check.rise + s->check.fall - 1)
Willy Tarreauef781042010-01-27 11:53:01 +0100488 sv_state = 3; /* UP */
489 else
490 sv_state = 2; /* going down */
491
Willy Tarreau892337c2014-05-13 23:41:20 +0200492 if (s->state == SRV_ST_STOPPING)
Willy Tarreauef781042010-01-27 11:53:01 +0100493 sv_state += 2;
494 } else {
Simon Horman125d0992013-02-24 17:23:38 +0900495 if (s->check.health)
Willy Tarreauef781042010-01-27 11:53:01 +0100496 sv_state = 1; /* going up */
497 else
498 sv_state = 0; /* DOWN */
499 }
500
Willy Tarreaua1dab552014-04-14 15:04:54 +0200501 hlen += snprintf(buffer + hlen, size - hlen,
Willy Tarreauef781042010-01-27 11:53:01 +0100502 srv_hlt_st[sv_state],
Willy Tarreau892337c2014-05-13 23:41:20 +0200503 (s->state != SRV_ST_STOPPED) ? (s->check.health - s->check.rise + 1) : (s->check.health),
504 (s->state != SRV_ST_STOPPED) ? (s->check.fall) : (s->check.rise));
Willy Tarreauef781042010-01-27 11:53:01 +0100505
Willy Tarreaua1dab552014-04-14 15:04:54 +0200506 hlen += snprintf(buffer + hlen, size - hlen, "; name=%s/%s; node=%s; weight=%d/%d; scur=%d/%d; qcur=%d",
Willy Tarreauef781042010-01-27 11:53:01 +0100507 s->proxy->id, s->id,
508 global.node,
509 (s->eweight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv,
510 (s->proxy->lbprm.tot_weight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv,
511 s->cur_sess, s->proxy->beconn - s->proxy->nbpend,
512 s->nbpend);
513
Willy Tarreau892337c2014-05-13 23:41:20 +0200514 if ((s->state == SRV_ST_STARTING) &&
Willy Tarreauef781042010-01-27 11:53:01 +0100515 now.tv_sec < s->last_change + s->slowstart &&
516 now.tv_sec >= s->last_change) {
517 ratio = MAX(1, 100 * (now.tv_sec - s->last_change) / s->slowstart);
Willy Tarreaua1dab552014-04-14 15:04:54 +0200518 hlen += snprintf(buffer + hlen, size - hlen, "; throttle=%d%%", ratio);
Willy Tarreauef781042010-01-27 11:53:01 +0100519 }
520
521 buffer[hlen++] = '\r';
522 buffer[hlen++] = '\n';
523
524 return hlen;
525}
526
Willy Tarreau20a18342013-12-05 00:31:46 +0100527/* Check the connection. If an error has already been reported or the socket is
528 * closed, keep errno intact as it is supposed to contain the valid error code.
529 * If no error is reported, check the socket's error queue using getsockopt().
530 * Warning, this must be done only once when returning from poll, and never
531 * after an I/O error was attempted, otherwise the error queue might contain
532 * inconsistent errors. If an error is detected, the CO_FL_ERROR is set on the
533 * socket. Returns non-zero if an error was reported, zero if everything is
534 * clean (including a properly closed socket).
535 */
536static int retrieve_errno_from_socket(struct connection *conn)
537{
538 int skerr;
539 socklen_t lskerr = sizeof(skerr);
540
541 if (conn->flags & CO_FL_ERROR && ((errno && errno != EAGAIN) || !conn->ctrl))
542 return 1;
543
Willy Tarreau3c728722014-01-23 13:50:42 +0100544 if (!conn_ctrl_ready(conn))
Willy Tarreau20a18342013-12-05 00:31:46 +0100545 return 0;
546
547 if (getsockopt(conn->t.sock.fd, SOL_SOCKET, SO_ERROR, &skerr, &lskerr) == 0)
548 errno = skerr;
549
550 if (errno == EAGAIN)
551 errno = 0;
552
553 if (!errno) {
554 /* we could not retrieve an error, that does not mean there is
555 * none. Just don't change anything and only report the prior
556 * error if any.
557 */
558 if (conn->flags & CO_FL_ERROR)
559 return 1;
560 else
561 return 0;
562 }
563
564 conn->flags |= CO_FL_ERROR | CO_FL_SOCK_WR_SH | CO_FL_SOCK_RD_SH;
565 return 1;
566}
567
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100568/* Try to collect as much information as possible on the connection status,
569 * and adjust the server status accordingly. It may make use of <errno_bck>
570 * if non-null when the caller is absolutely certain of its validity (eg:
571 * checked just after a syscall). If the caller doesn't have a valid errno,
572 * it can pass zero, and retrieve_errno_from_socket() will be called to try
573 * to extract errno from the socket. If no error is reported, it will consider
574 * the <expired> flag. This is intended to be used when a connection error was
575 * reported in conn->flags or when a timeout was reported in <expired>. The
576 * function takes care of not updating a server status which was already set.
577 * All situations where at least one of <expired> or CO_FL_ERROR are set
578 * produce a status.
579 */
580static void chk_report_conn_err(struct connection *conn, int errno_bck, int expired)
581{
582 struct check *check = conn->owner;
583 const char *err_msg;
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200584 struct chunk *chk;
Willy Tarreau90055f22014-10-02 14:51:02 +0200585 int step;
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100586
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100587 if (check->result != CHK_RES_UNKNOWN)
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100588 return;
589
590 errno = errno_bck;
591 if (!errno || errno == EAGAIN)
592 retrieve_errno_from_socket(conn);
593
594 if (!(conn->flags & CO_FL_ERROR) && !expired)
595 return;
596
597 /* we'll try to build a meaningful error message depending on the
598 * context of the error possibly present in conn->err_code, and the
599 * socket error possibly collected above. This is useful to know the
600 * exact step of the L6 layer (eg: SSL handshake).
601 */
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200602 chk = get_trash_chunk();
603
604 if (check->type == PR_O2_TCPCHK_CHK) {
Willy Tarreau90055f22014-10-02 14:51:02 +0200605 step = tcpcheck_get_step_id(check->server);
606 if (!step)
607 chunk_printf(chk, " at initial connection step of tcp-check");
608 else {
609 chunk_printf(chk, " at step %d of tcp-check", step);
610 /* we were looking for a string */
611 if (check->last_started_step && check->last_started_step->action == TCPCHK_ACT_CONNECT) {
612 if (check->last_started_step->port)
613 chunk_appendf(chk, " (connect port %d)" ,check->last_started_step->port);
614 else
615 chunk_appendf(chk, " (connect)");
616 }
617 else if (check->last_started_step && check->last_started_step->action == TCPCHK_ACT_EXPECT) {
618 if (check->last_started_step->string)
Baptiste Assmann68e4fc22015-05-01 08:09:29 +0200619 chunk_appendf(chk, " (expect string '%s')", check->last_started_step->string);
Willy Tarreau90055f22014-10-02 14:51:02 +0200620 else if (check->last_started_step->expect_regex)
621 chunk_appendf(chk, " (expect regex)");
622 }
623 else if (check->last_started_step && check->last_started_step->action == TCPCHK_ACT_SEND) {
624 chunk_appendf(chk, " (send)");
625 }
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200626 }
627 }
628
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100629 if (conn->err_code) {
630 if (errno && errno != EAGAIN)
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200631 chunk_printf(&trash, "%s (%s)%s", conn_err_code_str(conn), strerror(errno), chk->str);
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100632 else
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200633 chunk_printf(&trash, "%s%s", conn_err_code_str(conn), chk->str);
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100634 err_msg = trash.str;
635 }
636 else {
637 if (errno && errno != EAGAIN) {
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200638 chunk_printf(&trash, "%s%s", strerror(errno), chk->str);
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100639 err_msg = trash.str;
640 }
641 else {
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200642 err_msg = chk->str;
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100643 }
644 }
645
646 if ((conn->flags & (CO_FL_CONNECTED|CO_FL_WAIT_L4_CONN)) == CO_FL_WAIT_L4_CONN) {
647 /* L4 not established (yet) */
648 if (conn->flags & CO_FL_ERROR)
649 set_server_check_status(check, HCHK_STATUS_L4CON, err_msg);
650 else if (expired)
651 set_server_check_status(check, HCHK_STATUS_L4TOUT, err_msg);
652 }
653 else if ((conn->flags & (CO_FL_CONNECTED|CO_FL_WAIT_L6_CONN)) == CO_FL_WAIT_L6_CONN) {
654 /* L6 not established (yet) */
655 if (conn->flags & CO_FL_ERROR)
656 set_server_check_status(check, HCHK_STATUS_L6RSP, err_msg);
657 else if (expired)
658 set_server_check_status(check, HCHK_STATUS_L6TOUT, err_msg);
659 }
660 else if (conn->flags & CO_FL_ERROR) {
661 /* I/O error after connection was established and before we could diagnose */
662 set_server_check_status(check, HCHK_STATUS_SOCKERR, err_msg);
663 }
664 else if (expired) {
665 /* connection established but expired check */
666 if (check->type == PR_O2_SSL3_CHK)
667 set_server_check_status(check, HCHK_STATUS_L6TOUT, err_msg);
668 else /* HTTP, SMTP, ... */
669 set_server_check_status(check, HCHK_STATUS_L7TOUT, err_msg);
670 }
671
672 return;
673}
674
Willy Tarreaubaaee002006-06-26 02:48:02 +0200675/*
676 * This function is used only for server health-checks. It handles
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200677 * the connection acknowledgement. If the proxy requires L7 health-checks,
678 * it sends the request. In other cases, it calls set_server_check_status()
Simon Horman4a741432013-02-23 15:35:38 +0900679 * to set check->status, check->duration and check->result.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200680 */
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200681static void event_srv_chk_w(struct connection *conn)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200682{
Simon Horman4a741432013-02-23 15:35:38 +0900683 struct check *check = conn->owner;
684 struct server *s = check->server;
Simon Horman4a741432013-02-23 15:35:38 +0900685 struct task *t = check->task;
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200686
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100687 if (unlikely(check->result == CHK_RES_FAILED))
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100688 goto out_wakeup;
689
Willy Tarreau310987a2014-01-22 19:46:33 +0100690 if (conn->flags & CO_FL_HANDSHAKE)
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100691 return;
692
Willy Tarreau20a18342013-12-05 00:31:46 +0100693 if (retrieve_errno_from_socket(conn)) {
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100694 chk_report_conn_err(conn, errno, 0);
Willy Tarreau20a18342013-12-05 00:31:46 +0100695 __conn_data_stop_both(conn);
696 goto out_wakeup;
697 }
Krzysztof Piotr Oledzki6492db52010-01-02 22:03:01 +0100698
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100699 if (conn->flags & (CO_FL_SOCK_WR_SH | CO_FL_DATA_WR_SH)) {
Willy Tarreau20a18342013-12-05 00:31:46 +0100700 /* if the output is closed, we can't do anything */
701 conn->flags |= CO_FL_ERROR;
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100702 chk_report_conn_err(conn, 0, 0);
Willy Tarreau20a18342013-12-05 00:31:46 +0100703 goto out_wakeup;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200704 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200705
Willy Tarreau06559ac2013-12-05 01:53:08 +0100706 /* here, we know that the connection is established. That's enough for
707 * a pure TCP check.
708 */
709 if (!check->type)
710 goto out_wakeup;
711
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200712 if (check->type == PR_O2_TCPCHK_CHK) {
713 tcpcheck_main(conn);
714 return;
715 }
716
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100717 if (check->bo->o) {
Willy Tarreau1049b1f2014-02-02 01:51:17 +0100718 conn->xprt->snd_buf(conn, check->bo, 0);
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100719 if (conn->flags & CO_FL_ERROR) {
720 chk_report_conn_err(conn, errno, 0);
721 __conn_data_stop_both(conn);
722 goto out_wakeup;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200723 }
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100724 if (check->bo->o)
725 return;
726 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200727
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100728 /* full request sent, we allow up to <timeout.check> if nonzero for a response */
729 if (s->proxy->timeout.check) {
730 t->expire = tick_add_ifset(now_ms, s->proxy->timeout.check);
731 task_queue(t);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200732 }
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100733 goto out_nowake;
734
Willy Tarreau83749182007-04-15 20:56:27 +0200735 out_wakeup:
Willy Tarreaufdccded2008-08-29 18:19:04 +0200736 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreau83749182007-04-15 20:56:27 +0200737 out_nowake:
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200738 __conn_data_stop_send(conn); /* nothing more to write */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200739}
740
Willy Tarreaubaaee002006-06-26 02:48:02 +0200741/*
Willy Tarreauf3c69202006-07-09 16:42:34 +0200742 * This function is used only for server health-checks. It handles the server's
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +0200743 * reply to an HTTP request, SSL HELLO or MySQL client Auth. It calls
Simon Horman4a741432013-02-23 15:35:38 +0900744 * set_server_check_status() to update check->status, check->duration
745 * and check->result.
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200746
747 * The set_server_check_status function is called with HCHK_STATUS_L7OKD if
748 * an HTTP server replies HTTP 2xx or 3xx (valid responses), if an SMTP server
749 * returns 2xx, HCHK_STATUS_L6OK if an SSL server returns at least 5 bytes in
750 * response to an SSL HELLO (the principle is that this is enough to
751 * distinguish between an SSL server and a pure TCP relay). All other cases will
752 * call it with a proper error status like HCHK_STATUS_L7STS, HCHK_STATUS_L6RSP,
753 * etc.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200754 */
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200755static void event_srv_chk_r(struct connection *conn)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200756{
Simon Horman4a741432013-02-23 15:35:38 +0900757 struct check *check = conn->owner;
758 struct server *s = check->server;
759 struct task *t = check->task;
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200760 char *desc;
Willy Tarreau03938182010-03-17 21:52:07 +0100761 int done;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +0200762 unsigned short msglen;
Willy Tarreau83749182007-04-15 20:56:27 +0200763
Willy Tarreau6aaa1b82013-12-11 17:09:34 +0100764 if (unlikely(check->result == CHK_RES_FAILED))
Willy Tarreau83749182007-04-15 20:56:27 +0200765 goto out_wakeup;
Willy Tarreau83749182007-04-15 20:56:27 +0200766
Willy Tarreau310987a2014-01-22 19:46:33 +0100767 if (conn->flags & CO_FL_HANDSHAKE)
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200768 return;
Baptiste Assmann5ecb77f2013-10-06 23:24:13 +0200769
770 if (check->type == PR_O2_TCPCHK_CHK) {
771 tcpcheck_main(conn);
772 return;
773 }
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200774
Willy Tarreau83749182007-04-15 20:56:27 +0200775 /* Warning! Linux returns EAGAIN on SO_ERROR if data are still available
776 * but the connection was closed on the remote end. Fortunately, recv still
777 * works correctly and we don't need to do the getsockopt() on linux.
778 */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000779
780 /* Set buffer to point to the end of the data already read, and check
781 * that there is free space remaining. If the buffer is full, proceed
782 * with running the checks without attempting another socket read.
783 */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000784
Willy Tarreau03938182010-03-17 21:52:07 +0100785 done = 0;
Nick Chalk57b1bf72010-03-16 15:50:46 +0000786
Simon Horman4a741432013-02-23 15:35:38 +0900787 conn->xprt->rcv_buf(conn, check->bi, check->bi->size);
Willy Tarreauf1503172012-09-28 19:39:36 +0200788 if (conn->flags & (CO_FL_ERROR | CO_FL_SOCK_RD_SH | CO_FL_DATA_RD_SH)) {
Willy Tarreau03938182010-03-17 21:52:07 +0100789 done = 1;
Simon Horman4a741432013-02-23 15:35:38 +0900790 if ((conn->flags & CO_FL_ERROR) && !check->bi->i) {
Willy Tarreauf1503172012-09-28 19:39:36 +0200791 /* Report network errors only if we got no other data. Otherwise
792 * we'll let the upper layers decide whether the response is OK
793 * or not. It is very common that an RST sent by the server is
794 * reported as an error just after the last data chunk.
795 */
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100796 chk_report_conn_err(conn, errno, 0);
Willy Tarreauc1a07962010-03-16 20:55:43 +0100797 goto out_wakeup;
798 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200799 }
800
Willy Tarreau25e2ab52013-12-04 11:17:05 +0100801
Willy Tarreau03938182010-03-17 21:52:07 +0100802 /* Intermediate or complete response received.
Simon Horman4a741432013-02-23 15:35:38 +0900803 * Terminate string in check->bi->data buffer.
Willy Tarreau03938182010-03-17 21:52:07 +0100804 */
Simon Horman4a741432013-02-23 15:35:38 +0900805 if (check->bi->i < check->bi->size)
806 check->bi->data[check->bi->i] = '\0';
Willy Tarreau03938182010-03-17 21:52:07 +0100807 else {
Simon Horman4a741432013-02-23 15:35:38 +0900808 check->bi->data[check->bi->i - 1] = '\0';
Willy Tarreau03938182010-03-17 21:52:07 +0100809 done = 1; /* buffer full, don't wait for more data */
810 }
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200811
Nick Chalk57b1bf72010-03-16 15:50:46 +0000812 /* Run the checks... */
Simon Horman4a741432013-02-23 15:35:38 +0900813 switch (check->type) {
Willy Tarreau1620ec32011-08-06 17:05:02 +0200814 case PR_O2_HTTP_CHK:
Simon Horman4a741432013-02-23 15:35:38 +0900815 if (!done && check->bi->i < strlen("HTTP/1.0 000\r"))
Willy Tarreau03938182010-03-17 21:52:07 +0100816 goto wait_more_data;
817
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100818 /* Check if the server speaks HTTP 1.X */
Simon Horman4a741432013-02-23 15:35:38 +0900819 if ((check->bi->i < strlen("HTTP/1.0 000\r")) ||
820 (memcmp(check->bi->data, "HTTP/1.", 7) != 0 ||
821 (*(check->bi->data + 12) != ' ' && *(check->bi->data + 12) != '\r')) ||
822 !isdigit((unsigned char) *(check->bi->data + 9)) || !isdigit((unsigned char) *(check->bi->data + 10)) ||
823 !isdigit((unsigned char) *(check->bi->data + 11))) {
824 cut_crlf(check->bi->data);
825 set_server_check_status(check, HCHK_STATUS_L7RSP, check->bi->data);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200826
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100827 goto out_wakeup;
828 }
829
Simon Horman4a741432013-02-23 15:35:38 +0900830 check->code = str2uic(check->bi->data + 9);
831 desc = ltrim(check->bi->data + 12, ' ');
Nick Chalk57b1bf72010-03-16 15:50:46 +0000832
Willy Tarreaubd741542010-03-16 18:46:54 +0100833 if ((s->proxy->options & PR_O_DISABLE404) &&
Willy Tarreau892337c2014-05-13 23:41:20 +0200834 (s->state != SRV_ST_STOPPED) && (check->code == 404)) {
Nick Chalk57b1bf72010-03-16 15:50:46 +0000835 /* 404 may be accepted as "stopping" only if the server was up */
836 cut_crlf(desc);
Simon Horman4a741432013-02-23 15:35:38 +0900837 set_server_check_status(check, HCHK_STATUS_L7OKCD, desc);
Nick Chalk57b1bf72010-03-16 15:50:46 +0000838 }
Willy Tarreaubd741542010-03-16 18:46:54 +0100839 else if (s->proxy->options2 & PR_O2_EXP_TYPE) {
840 /* Run content verification check... We know we have at least 13 chars */
841 if (!httpchk_expect(s, done))
842 goto wait_more_data;
843 }
844 /* check the reply : HTTP/1.X 2xx and 3xx are OK */
Simon Horman4a741432013-02-23 15:35:38 +0900845 else if (*(check->bi->data + 9) == '2' || *(check->bi->data + 9) == '3') {
Willy Tarreaubd741542010-03-16 18:46:54 +0100846 cut_crlf(desc);
Simon Horman4a741432013-02-23 15:35:38 +0900847 set_server_check_status(check, HCHK_STATUS_L7OKD, desc);
Willy Tarreaubd741542010-03-16 18:46:54 +0100848 }
Nick Chalk57b1bf72010-03-16 15:50:46 +0000849 else {
850 cut_crlf(desc);
Simon Horman4a741432013-02-23 15:35:38 +0900851 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Nick Chalk57b1bf72010-03-16 15:50:46 +0000852 }
Willy Tarreau1620ec32011-08-06 17:05:02 +0200853 break;
854
855 case PR_O2_SSL3_CHK:
Simon Horman4a741432013-02-23 15:35:38 +0900856 if (!done && check->bi->i < 5)
Willy Tarreau03938182010-03-17 21:52:07 +0100857 goto wait_more_data;
858
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100859 /* Check for SSLv3 alert or handshake */
Simon Horman4a741432013-02-23 15:35:38 +0900860 if ((check->bi->i >= 5) && (*check->bi->data == 0x15 || *check->bi->data == 0x16))
861 set_server_check_status(check, HCHK_STATUS_L6OK, NULL);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200862 else
Simon Horman4a741432013-02-23 15:35:38 +0900863 set_server_check_status(check, HCHK_STATUS_L6RSP, NULL);
Willy Tarreau1620ec32011-08-06 17:05:02 +0200864 break;
865
866 case PR_O2_SMTP_CHK:
Simon Horman4a741432013-02-23 15:35:38 +0900867 if (!done && check->bi->i < strlen("000\r"))
Willy Tarreau03938182010-03-17 21:52:07 +0100868 goto wait_more_data;
869
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200870 /* Check if the server speaks SMTP */
Simon Horman4a741432013-02-23 15:35:38 +0900871 if ((check->bi->i < strlen("000\r")) ||
872 (*(check->bi->data + 3) != ' ' && *(check->bi->data + 3) != '\r') ||
873 !isdigit((unsigned char) *check->bi->data) || !isdigit((unsigned char) *(check->bi->data + 1)) ||
874 !isdigit((unsigned char) *(check->bi->data + 2))) {
875 cut_crlf(check->bi->data);
876 set_server_check_status(check, HCHK_STATUS_L7RSP, check->bi->data);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200877
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200878 goto out_wakeup;
879 }
880
Simon Horman4a741432013-02-23 15:35:38 +0900881 check->code = str2uic(check->bi->data);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200882
Simon Horman4a741432013-02-23 15:35:38 +0900883 desc = ltrim(check->bi->data + 3, ' ');
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200884 cut_crlf(desc);
885
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100886 /* Check for SMTP code 2xx (should be 250) */
Simon Horman4a741432013-02-23 15:35:38 +0900887 if (*check->bi->data == '2')
888 set_server_check_status(check, HCHK_STATUS_L7OKD, desc);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200889 else
Simon Horman4a741432013-02-23 15:35:38 +0900890 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Willy Tarreau1620ec32011-08-06 17:05:02 +0200891 break;
892
Simon Hormana2b9dad2013-02-12 10:45:54 +0900893 case PR_O2_LB_AGENT_CHK: {
Willy Tarreau81f5d942013-12-09 20:51:51 +0100894 int status = HCHK_STATUS_CHECKED;
895 const char *hs = NULL; /* health status */
896 const char *as = NULL; /* admin status */
897 const char *ps = NULL; /* performance status */
898 const char *err = NULL; /* first error to report */
899 const char *wrn = NULL; /* first warning to report */
900 char *cmd, *p;
Simon Hormana2b9dad2013-02-12 10:45:54 +0900901
Willy Tarreau81f5d942013-12-09 20:51:51 +0100902 /* We're getting an agent check response. The agent could
903 * have been disabled in the mean time with a long check
904 * still pending. It is important that we ignore the whole
905 * response.
906 */
907 if (!(check->server->agent.state & CHK_ST_ENABLED))
908 break;
909
910 /* The agent supports strings made of a single line ended by the
911 * first CR ('\r') or LF ('\n'). This line is composed of words
912 * delimited by spaces (' '), tabs ('\t'), or commas (','). The
913 * line may optionally contained a description of a state change
914 * after a sharp ('#'), which is only considered if a health state
915 * is announced.
916 *
917 * Words may be composed of :
918 * - a numeric weight suffixed by the percent character ('%').
919 * - a health status among "up", "down", "stopped", and "fail".
920 * - an admin status among "ready", "drain", "maint".
921 *
922 * These words may appear in any order. If multiple words of the
923 * same category appear, the last one wins.
924 */
925
Willy Tarreau9809b782013-12-11 21:40:11 +0100926 p = check->bi->data;
927 while (*p && *p != '\n' && *p != '\r')
928 p++;
929
930 if (!*p) {
931 if (!done)
932 goto wait_more_data;
Simon Hormana2b9dad2013-02-12 10:45:54 +0900933
Willy Tarreau9809b782013-12-11 21:40:11 +0100934 /* at least inform the admin that the agent is mis-behaving */
935 set_server_check_status(check, check->status, "Ignoring incomplete line from agent");
936 break;
937 }
Willy Tarreau81f5d942013-12-09 20:51:51 +0100938
Willy Tarreau9809b782013-12-11 21:40:11 +0100939 *p = 0;
Willy Tarreau81f5d942013-12-09 20:51:51 +0100940 cmd = check->bi->data;
Simon Hormana2b9dad2013-02-12 10:45:54 +0900941
Willy Tarreau81f5d942013-12-09 20:51:51 +0100942 while (*cmd) {
943 /* look for next word */
944 if (*cmd == ' ' || *cmd == '\t' || *cmd == ',') {
945 cmd++;
946 continue;
947 }
Simon Horman671b6f02013-11-25 10:46:39 +0900948
Willy Tarreau81f5d942013-12-09 20:51:51 +0100949 if (*cmd == '#') {
950 /* this is the beginning of a health status description,
951 * skip the sharp and blanks.
952 */
953 cmd++;
954 while (*cmd == '\t' || *cmd == ' ')
955 cmd++;
Simon Horman671b6f02013-11-25 10:46:39 +0900956 break;
Simon Hormana2b9dad2013-02-12 10:45:54 +0900957 }
Willy Tarreau81f5d942013-12-09 20:51:51 +0100958
959 /* find the end of the word so that we have a null-terminated
960 * word between <cmd> and <p>.
961 */
962 p = cmd + 1;
963 while (*p && *p != '\t' && *p != ' ' && *p != '\n' && *p != ',')
964 p++;
965 if (*p)
966 *p++ = 0;
967
968 /* first, health statuses */
969 if (strcasecmp(cmd, "up") == 0) {
970 check->health = check->rise + check->fall - 1;
Simon Hormana2b9dad2013-02-12 10:45:54 +0900971 status = HCHK_STATUS_L7OKD;
Willy Tarreau81f5d942013-12-09 20:51:51 +0100972 hs = cmd;
973 }
974 else if (strcasecmp(cmd, "down") == 0) {
975 check->health = 0;
976 status = HCHK_STATUS_L7STS;
977 hs = cmd;
Simon Hormana2b9dad2013-02-12 10:45:54 +0900978 }
Willy Tarreau81f5d942013-12-09 20:51:51 +0100979 else if (strcasecmp(cmd, "stopped") == 0) {
980 check->health = 0;
981 status = HCHK_STATUS_L7STS;
982 hs = cmd;
983 }
984 else if (strcasecmp(cmd, "fail") == 0) {
985 check->health = 0;
986 status = HCHK_STATUS_L7STS;
987 hs = cmd;
988 }
989 /* admin statuses */
990 else if (strcasecmp(cmd, "ready") == 0) {
991 as = cmd;
992 }
993 else if (strcasecmp(cmd, "drain") == 0) {
994 as = cmd;
995 }
996 else if (strcasecmp(cmd, "maint") == 0) {
997 as = cmd;
998 }
999 /* else try to parse a weight here and keep the last one */
1000 else if (isdigit((unsigned char)*cmd) && strchr(cmd, '%') != NULL) {
1001 ps = cmd;
1002 }
1003 else {
1004 /* keep a copy of the first error */
1005 if (!err)
1006 err = cmd;
1007 }
1008 /* skip to next word */
1009 cmd = p;
1010 }
1011 /* here, cmd points either to \0 or to the beginning of a
1012 * description. Skip possible leading spaces.
1013 */
1014 while (*cmd == ' ' || *cmd == '\n')
1015 cmd++;
1016
1017 /* First, update the admin status so that we avoid sending other
1018 * possibly useless warnings and can also update the health if
1019 * present after going back up.
1020 */
1021 if (as) {
1022 if (strcasecmp(as, "drain") == 0)
1023 srv_adm_set_drain(check->server);
1024 else if (strcasecmp(as, "maint") == 0)
1025 srv_adm_set_maint(check->server);
1026 else
1027 srv_adm_set_ready(check->server);
Simon Hormana2b9dad2013-02-12 10:45:54 +09001028 }
1029
Willy Tarreau81f5d942013-12-09 20:51:51 +01001030 /* now change weights */
1031 if (ps) {
1032 const char *msg;
1033
1034 msg = server_parse_weight_change_request(s, ps);
1035 if (!wrn || !*wrn)
1036 wrn = msg;
1037 }
1038
1039 /* and finally health status */
1040 if (hs) {
1041 /* We'll report some of the warnings and errors we have
1042 * here. Down reports are critical, we leave them untouched.
1043 * Lack of report, or report of 'UP' leaves the room for
1044 * ERR first, then WARN.
Simon Hormana2b9dad2013-02-12 10:45:54 +09001045 */
Willy Tarreau81f5d942013-12-09 20:51:51 +01001046 const char *msg = cmd;
1047 struct chunk *t;
1048
1049 if (!*msg || status == HCHK_STATUS_L7OKD) {
1050 if (err && *err)
1051 msg = err;
1052 else if (wrn && *wrn)
1053 msg = wrn;
Simon Hormana2b9dad2013-02-12 10:45:54 +09001054 }
Willy Tarreau81f5d942013-12-09 20:51:51 +01001055
1056 t = get_trash_chunk();
1057 chunk_printf(t, "via agent : %s%s%s%s",
1058 hs, *msg ? " (" : "",
1059 msg, *msg ? ")" : "");
1060
1061 set_server_check_status(check, status, t->str);
Simon Hormana2b9dad2013-02-12 10:45:54 +09001062 }
Willy Tarreau81f5d942013-12-09 20:51:51 +01001063 else if (err && *err) {
1064 /* No status change but we'd like to report something odd.
1065 * Just report the current state and copy the message.
1066 */
1067 chunk_printf(&trash, "agent reports an error : %s", err);
1068 set_server_check_status(check, status/*check->status*/, trash.str);
Simon Hormana2b9dad2013-02-12 10:45:54 +09001069
Willy Tarreau81f5d942013-12-09 20:51:51 +01001070 }
1071 else if (wrn && *wrn) {
1072 /* No status change but we'd like to report something odd.
1073 * Just report the current state and copy the message.
1074 */
1075 chunk_printf(&trash, "agent warns : %s", wrn);
1076 set_server_check_status(check, status/*check->status*/, trash.str);
1077 }
1078 else
1079 set_server_check_status(check, status, NULL);
Simon Hormana2b9dad2013-02-12 10:45:54 +09001080 break;
1081 }
1082
Willy Tarreau1620ec32011-08-06 17:05:02 +02001083 case PR_O2_PGSQL_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001084 if (!done && check->bi->i < 9)
Rauf Kuliyev38b41562011-01-04 15:14:13 +01001085 goto wait_more_data;
1086
Simon Horman4a741432013-02-23 15:35:38 +09001087 if (check->bi->data[0] == 'R') {
1088 set_server_check_status(check, HCHK_STATUS_L7OKD, "PostgreSQL server is ok");
Rauf Kuliyev38b41562011-01-04 15:14:13 +01001089 }
1090 else {
Simon Horman4a741432013-02-23 15:35:38 +09001091 if ((check->bi->data[0] == 'E') && (check->bi->data[5]!=0) && (check->bi->data[6]!=0))
1092 desc = &check->bi->data[6];
Rauf Kuliyev38b41562011-01-04 15:14:13 +01001093 else
1094 desc = "PostgreSQL unknown error";
1095
Simon Horman4a741432013-02-23 15:35:38 +09001096 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Rauf Kuliyev38b41562011-01-04 15:14:13 +01001097 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001098 break;
1099
1100 case PR_O2_REDIS_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001101 if (!done && check->bi->i < 7)
Hervé COMMOWICKec032d62011-08-05 16:23:48 +02001102 goto wait_more_data;
1103
Simon Horman4a741432013-02-23 15:35:38 +09001104 if (strcmp(check->bi->data, "+PONG\r\n") == 0) {
1105 set_server_check_status(check, HCHK_STATUS_L7OKD, "Redis server is ok");
Hervé COMMOWICKec032d62011-08-05 16:23:48 +02001106 }
1107 else {
Simon Horman4a741432013-02-23 15:35:38 +09001108 set_server_check_status(check, HCHK_STATUS_L7STS, check->bi->data);
Hervé COMMOWICKec032d62011-08-05 16:23:48 +02001109 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001110 break;
1111
1112 case PR_O2_MYSQL_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001113 if (!done && check->bi->i < 5)
Willy Tarreau03938182010-03-17 21:52:07 +01001114 goto wait_more_data;
1115
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001116 if (s->proxy->check_len == 0) { // old mode
Simon Horman4a741432013-02-23 15:35:38 +09001117 if (*(check->bi->data + 4) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001118 /* We set the MySQL Version in description for information purpose
1119 * FIXME : it can be cool to use MySQL Version for other purpose,
1120 * like mark as down old MySQL server.
1121 */
Simon Horman4a741432013-02-23 15:35:38 +09001122 if (check->bi->i > 51) {
1123 desc = ltrim(check->bi->data + 5, ' ');
1124 set_server_check_status(check, HCHK_STATUS_L7OKD, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001125 }
1126 else {
1127 if (!done)
1128 goto wait_more_data;
1129 /* it seems we have a OK packet but without a valid length,
1130 * it must be a protocol error
1131 */
Simon Horman4a741432013-02-23 15:35:38 +09001132 set_server_check_status(check, HCHK_STATUS_L7RSP, check->bi->data);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001133 }
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001134 }
1135 else {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001136 /* An error message is attached in the Error packet */
Simon Horman4a741432013-02-23 15:35:38 +09001137 desc = ltrim(check->bi->data + 7, ' ');
1138 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001139 }
1140 } else {
Simon Horman4a741432013-02-23 15:35:38 +09001141 unsigned int first_packet_len = ((unsigned int) *check->bi->data) +
1142 (((unsigned int) *(check->bi->data + 1)) << 8) +
1143 (((unsigned int) *(check->bi->data + 2)) << 16);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001144
Simon Horman4a741432013-02-23 15:35:38 +09001145 if (check->bi->i == first_packet_len + 4) {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001146 /* MySQL Error packet always begin with field_count = 0xff */
Simon Horman4a741432013-02-23 15:35:38 +09001147 if (*(check->bi->data + 4) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001148 /* We have only one MySQL packet and it is a Handshake Initialization packet
1149 * but we need to have a second packet to know if it is alright
1150 */
Simon Horman4a741432013-02-23 15:35:38 +09001151 if (!done && check->bi->i < first_packet_len + 5)
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001152 goto wait_more_data;
1153 }
1154 else {
1155 /* We have only one packet and it is an Error packet,
1156 * an error message is attached, so we can display it
1157 */
Simon Horman4a741432013-02-23 15:35:38 +09001158 desc = &check->bi->data[7];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001159 //Warning("onlyoneERR: %s\n", desc);
Simon Horman4a741432013-02-23 15:35:38 +09001160 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001161 }
Simon Horman4a741432013-02-23 15:35:38 +09001162 } else if (check->bi->i > first_packet_len + 4) {
1163 unsigned int second_packet_len = ((unsigned int) *(check->bi->data + first_packet_len + 4)) +
1164 (((unsigned int) *(check->bi->data + first_packet_len + 5)) << 8) +
1165 (((unsigned int) *(check->bi->data + first_packet_len + 6)) << 16);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001166
Simon Horman4a741432013-02-23 15:35:38 +09001167 if (check->bi->i == first_packet_len + 4 + second_packet_len + 4 ) {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001168 /* We have 2 packets and that's good */
1169 /* Check if the second packet is a MySQL Error packet or not */
Simon Horman4a741432013-02-23 15:35:38 +09001170 if (*(check->bi->data + first_packet_len + 8) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001171 /* No error packet */
1172 /* We set the MySQL Version in description for information purpose */
Simon Horman4a741432013-02-23 15:35:38 +09001173 desc = &check->bi->data[5];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001174 //Warning("2packetOK: %s\n", desc);
Simon Horman4a741432013-02-23 15:35:38 +09001175 set_server_check_status(check, HCHK_STATUS_L7OKD, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001176 }
1177 else {
1178 /* An error message is attached in the Error packet
1179 * so we can display it ! :)
1180 */
Simon Horman4a741432013-02-23 15:35:38 +09001181 desc = &check->bi->data[first_packet_len+11];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001182 //Warning("2packetERR: %s\n", desc);
Simon Horman4a741432013-02-23 15:35:38 +09001183 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001184 }
1185 }
1186 }
1187 else {
Willy Tarreau03938182010-03-17 21:52:07 +01001188 if (!done)
1189 goto wait_more_data;
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001190 /* it seems we have a Handshake Initialization packet but without a valid length,
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001191 * it must be a protocol error
1192 */
Simon Horman4a741432013-02-23 15:35:38 +09001193 desc = &check->bi->data[5];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001194 //Warning("protoerr: %s\n", desc);
Simon Horman4a741432013-02-23 15:35:38 +09001195 set_server_check_status(check, HCHK_STATUS_L7RSP, desc);
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001196 }
1197 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001198 break;
1199
1200 case PR_O2_LDAP_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001201 if (!done && check->bi->i < 14)
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001202 goto wait_more_data;
1203
1204 /* Check if the server speaks LDAP (ASN.1/BER)
1205 * http://en.wikipedia.org/wiki/Basic_Encoding_Rules
1206 * http://tools.ietf.org/html/rfc4511
1207 */
1208
1209 /* http://tools.ietf.org/html/rfc4511#section-4.1.1
1210 * LDAPMessage: 0x30: SEQUENCE
1211 */
Simon Horman4a741432013-02-23 15:35:38 +09001212 if ((check->bi->i < 14) || (*(check->bi->data) != '\x30')) {
1213 set_server_check_status(check, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001214 }
1215 else {
1216 /* size of LDAPMessage */
Simon Horman4a741432013-02-23 15:35:38 +09001217 msglen = (*(check->bi->data + 1) & 0x80) ? (*(check->bi->data + 1) & 0x7f) : 0;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001218
1219 /* http://tools.ietf.org/html/rfc4511#section-4.2.2
1220 * messageID: 0x02 0x01 0x01: INTEGER 1
1221 * protocolOp: 0x61: bindResponse
1222 */
1223 if ((msglen > 2) ||
Simon Horman4a741432013-02-23 15:35:38 +09001224 (memcmp(check->bi->data + 2 + msglen, "\x02\x01\x01\x61", 4) != 0)) {
1225 set_server_check_status(check, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001226
1227 goto out_wakeup;
1228 }
1229
1230 /* size of bindResponse */
Simon Horman4a741432013-02-23 15:35:38 +09001231 msglen += (*(check->bi->data + msglen + 6) & 0x80) ? (*(check->bi->data + msglen + 6) & 0x7f) : 0;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001232
1233 /* http://tools.ietf.org/html/rfc4511#section-4.1.9
1234 * ldapResult: 0x0a 0x01: ENUMERATION
1235 */
1236 if ((msglen > 4) ||
Simon Horman4a741432013-02-23 15:35:38 +09001237 (memcmp(check->bi->data + 7 + msglen, "\x0a\x01", 2) != 0)) {
1238 set_server_check_status(check, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001239
1240 goto out_wakeup;
1241 }
1242
1243 /* http://tools.ietf.org/html/rfc4511#section-4.1.9
1244 * resultCode
1245 */
Simon Horman4a741432013-02-23 15:35:38 +09001246 check->code = *(check->bi->data + msglen + 9);
1247 if (check->code) {
1248 set_server_check_status(check, HCHK_STATUS_L7STS, "See RFC: http://tools.ietf.org/html/rfc4511#section-4.1.9");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001249 } else {
Simon Horman4a741432013-02-23 15:35:38 +09001250 set_server_check_status(check, HCHK_STATUS_L7OKD, "Success");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001251 }
1252 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001253 break;
1254
1255 default:
Willy Tarreau06559ac2013-12-05 01:53:08 +01001256 /* for other checks (eg: pure TCP), delegate to the main task */
Willy Tarreau1620ec32011-08-06 17:05:02 +02001257 break;
1258 } /* switch */
Willy Tarreau83749182007-04-15 20:56:27 +02001259
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001260 out_wakeup:
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001261 /* collect possible new errors */
1262 if (conn->flags & CO_FL_ERROR)
1263 chk_report_conn_err(conn, 0, 0);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001264
Nick Chalk57b1bf72010-03-16 15:50:46 +00001265 /* Reset the check buffer... */
Simon Horman4a741432013-02-23 15:35:38 +09001266 *check->bi->data = '\0';
1267 check->bi->i = 0;
Nick Chalk57b1bf72010-03-16 15:50:46 +00001268
Willy Tarreaufd29cc52012-11-23 09:18:20 +01001269 /* Close the connection... We absolutely want to perform a hard close
1270 * and reset the connection if some data are pending, otherwise we end
1271 * up with many TIME_WAITs and eat all the source port range quickly.
1272 * To avoid sending RSTs all the time, we first try to drain pending
1273 * data.
1274 */
Willy Tarreauf1503172012-09-28 19:39:36 +02001275 if (conn->xprt && conn->xprt->shutw)
1276 conn->xprt->shutw(conn, 0);
Willy Tarreau2b57cb82013-06-10 19:56:38 +02001277
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001278 /* OK, let's not stay here forever */
Willy Tarreau6aaa1b82013-12-11 17:09:34 +01001279 if (check->result == CHK_RES_FAILED)
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001280 conn->flags |= CO_FL_ERROR;
1281
Willy Tarreaua522f802012-11-23 08:56:35 +01001282 __conn_data_stop_both(conn);
Willy Tarreaufdccded2008-08-29 18:19:04 +02001283 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreau3267d362012-08-17 23:53:56 +02001284 return;
Willy Tarreau03938182010-03-17 21:52:07 +01001285
1286 wait_more_data:
Willy Tarreauf817e9f2014-01-10 16:58:45 +01001287 __conn_data_want_recv(conn);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001288}
1289
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001290/*
1291 * This function is used only for server health-checks. It handles connection
1292 * status updates including errors. If necessary, it wakes the check task up.
1293 * It always returns 0.
1294 */
1295static int wake_srv_chk(struct connection *conn)
Willy Tarreau20bea422012-07-06 12:00:49 +02001296{
Simon Horman4a741432013-02-23 15:35:38 +09001297 struct check *check = conn->owner;
Willy Tarreau20bea422012-07-06 12:00:49 +02001298
Willy Tarreau6c560da2012-11-24 11:14:45 +01001299 if (unlikely(conn->flags & CO_FL_ERROR)) {
Willy Tarreau02b0f582013-12-03 15:42:33 +01001300 /* We may get error reports bypassing the I/O handlers, typically
1301 * the case when sending a pure TCP check which fails, then the I/O
1302 * handlers above are not called. This is completely handled by the
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001303 * main processing task so let's simply wake it up. If we get here,
1304 * we expect errno to still be valid.
1305 */
1306 chk_report_conn_err(conn, errno, 0);
1307
Willy Tarreau2d351b62013-12-05 02:36:25 +01001308 __conn_data_stop_both(conn);
1309 task_wakeup(check->task, TASK_WOKEN_IO);
1310 }
Willy Tarreau3be293f2014-02-05 18:31:24 +01001311 else if (!(conn->flags & (CO_FL_DATA_RD_ENA|CO_FL_DATA_WR_ENA|CO_FL_HANDSHAKE))) {
1312 /* we may get here if only a connection probe was required : we
1313 * don't have any data to send nor anything expected in response,
1314 * so the completion of the connection establishment is enough.
1315 */
1316 task_wakeup(check->task, TASK_WOKEN_IO);
1317 }
Willy Tarreau2d351b62013-12-05 02:36:25 +01001318
Willy Tarreau6aaa1b82013-12-11 17:09:34 +01001319 if (check->result != CHK_RES_UNKNOWN) {
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001320 /* We're here because nobody wants to handle the error, so we
1321 * sure want to abort the hard way.
Willy Tarreau02b0f582013-12-03 15:42:33 +01001322 */
Willy Tarreau46be2e52014-01-20 12:10:52 +01001323 conn_drain(conn);
Willy Tarreauf79c8172013-10-21 16:30:56 +02001324 conn_force_close(conn);
Willy Tarreau2d351b62013-12-05 02:36:25 +01001325 }
Willy Tarreau3267d362012-08-17 23:53:56 +02001326 return 0;
Willy Tarreau20bea422012-07-06 12:00:49 +02001327}
1328
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001329struct data_cb check_conn_cb = {
1330 .recv = event_srv_chk_r,
1331 .send = event_srv_chk_w,
1332 .wake = wake_srv_chk,
1333};
1334
Willy Tarreaubaaee002006-06-26 02:48:02 +02001335/*
Willy Tarreau2e993902011-10-31 11:53:20 +01001336 * updates the server's weight during a warmup stage. Once the final weight is
1337 * reached, the task automatically stops. Note that any server status change
1338 * must have updated s->last_change accordingly.
1339 */
1340static struct task *server_warmup(struct task *t)
1341{
1342 struct server *s = t->context;
1343
1344 /* by default, plan on stopping the task */
1345 t->expire = TICK_ETERNITY;
Willy Tarreau20125212014-05-13 19:44:56 +02001346 if ((s->admin & SRV_ADMF_MAINT) ||
Willy Tarreau892337c2014-05-13 23:41:20 +02001347 (s->state != SRV_ST_STARTING))
Willy Tarreau2e993902011-10-31 11:53:20 +01001348 return t;
1349
Willy Tarreau892337c2014-05-13 23:41:20 +02001350 /* recalculate the weights and update the state */
Willy Tarreau004e0452013-11-21 11:22:01 +01001351 server_recalc_eweight(s);
Willy Tarreau2e993902011-10-31 11:53:20 +01001352
1353 /* probably that we can refill this server with a bit more connections */
Willy Tarreau4aac7db2014-05-16 11:48:10 +02001354 pendconn_grab_from_px(s);
Willy Tarreau2e993902011-10-31 11:53:20 +01001355
1356 /* get back there in 1 second or 1/20th of the slowstart interval,
1357 * whichever is greater, resulting in small 5% steps.
1358 */
Willy Tarreau892337c2014-05-13 23:41:20 +02001359 if (s->state == SRV_ST_STARTING)
Willy Tarreau2e993902011-10-31 11:53:20 +01001360 t->expire = tick_add(now_ms, MS_TO_TICKS(MAX(1000, s->slowstart / 20)));
1361 return t;
1362}
1363
1364/*
Simon Hormanb00d17a2014-06-13 16:18:16 +09001365 * establish a server health-check.
1366 *
1367 * It can return one of :
1368 * - SN_ERR_NONE if everything's OK and tcpcheck_main() was not called
1369 * - SN_ERR_UP if if everything's OK and tcpcheck_main() was called
1370 * - SN_ERR_SRVTO if there are no more servers
1371 * - SN_ERR_SRVCL if the connection was refused by the server
1372 * - SN_ERR_PRXCOND if the connection has been limited by the proxy (maxconn)
1373 * - SN_ERR_RESOURCE if a system resource is lacking (eg: fd limits, ports, ...)
1374 * - SN_ERR_INTERNAL for any other purely internal errors
1375 * Additionnally, in the case of SN_ERR_RESOURCE, an emergency log will be emitted.
1376 * Note that we try to prevent the network stack from sending the ACK during the
1377 * connect() when a pure TCP check is used (without PROXY protocol).
1378 */
1379static int connect_chk(struct task *t)
1380{
1381 struct check *check = t->context;
1382 struct server *s = check->server;
1383 struct connection *conn = check->conn;
1384 struct protocol *proto;
1385 int ret;
Willy Tarreau02c62122014-12-08 12:11:28 +01001386 int quickack;
Simon Hormanb00d17a2014-06-13 16:18:16 +09001387
1388 /* tcpcheck send/expect initialisation */
1389 if (check->type == PR_O2_TCPCHK_CHK)
1390 check->current_step = NULL;
1391
1392 /* prepare the check buffer.
1393 * This should not be used if check is the secondary agent check
1394 * of a server as s->proxy->check_req will relate to the
1395 * configuration of the primary check. Similarly, tcp-check uses
1396 * its own strings.
1397 */
1398 if (check->type && check->type != PR_O2_TCPCHK_CHK && !(check->state & CHK_ST_AGENT)) {
1399 bo_putblk(check->bo, s->proxy->check_req, s->proxy->check_len);
1400
1401 /* we want to check if this host replies to HTTP or SSLv3 requests
1402 * so we'll send the request, and won't wake the checker up now.
1403 */
1404 if ((check->type) == PR_O2_SSL3_CHK) {
1405 /* SSL requires that we put Unix time in the request */
1406 int gmt_time = htonl(date.tv_sec);
1407 memcpy(check->bo->data + 11, &gmt_time, 4);
1408 }
1409 else if ((check->type) == PR_O2_HTTP_CHK) {
1410 if (s->proxy->options2 & PR_O2_CHK_SNDST)
1411 bo_putblk(check->bo, trash.str, httpchk_build_status_header(s, trash.str, trash.size));
Cyril Bontéa448e162015-01-30 00:07:07 +01001412 /* prevent HTTP keep-alive when "http-check expect" is used */
1413 if (s->proxy->options2 & PR_O2_EXP_TYPE)
1414 bo_putstr(check->bo, "Connection: close\r\n");
Simon Hormanb00d17a2014-06-13 16:18:16 +09001415 bo_putstr(check->bo, "\r\n");
1416 *check->bo->p = '\0'; /* to make gdb output easier to read */
1417 }
1418 }
1419
1420 /* prepare a new connection */
1421 conn_init(conn);
Cyril Bonté1f96a872014-11-15 22:41:27 +01001422 conn_prepare(conn, s->check_common.proto, check->xprt);
Simon Hormanb00d17a2014-06-13 16:18:16 +09001423 conn_attach(conn, check, &check_conn_cb);
1424 conn->target = &s->obj_type;
1425
1426 /* no client address */
1427 clear_addr(&conn->addr.from);
1428
1429 if (is_addr(&s->check_common.addr)) {
1430
1431 /* we'll connect to the check addr specified on the server */
1432 conn->addr.to = s->check_common.addr;
1433 proto = s->check_common.proto;
1434 }
1435 else {
1436 /* we'll connect to the addr on the server */
1437 conn->addr.to = s->addr;
1438 proto = s->proto;
1439 }
1440
1441 if (check->port) {
1442 set_host_port(&conn->addr.to, check->port);
1443 }
1444
Willy Tarreau02c62122014-12-08 12:11:28 +01001445 /* only plain tcp-check supports quick ACK */
1446 quickack = check->type == 0 || check->type == PR_O2_TCPCHK_CHK;
1447
Willy Tarreau619a6ae2014-12-08 11:52:28 +01001448 if (check->type == PR_O2_TCPCHK_CHK && !LIST_ISEMPTY(&s->proxy->tcpcheck_rules)) {
Willy Tarreau5bff0592015-05-13 12:24:53 +02001449 struct tcpcheck_rule *r;
1450
1451 r = LIST_NEXT(&s->proxy->tcpcheck_rules, struct tcpcheck_rule *, list);
1452
Simon Hormanb00d17a2014-06-13 16:18:16 +09001453 /* if first step is a 'connect', then tcpcheck_main must run it */
1454 if (r->action == TCPCHK_ACT_CONNECT) {
1455 tcpcheck_main(conn);
1456 return SN_ERR_UP;
1457 }
Willy Tarreau02c62122014-12-08 12:11:28 +01001458 if (r->action == TCPCHK_ACT_EXPECT)
1459 quickack = 0;
Simon Hormanb00d17a2014-06-13 16:18:16 +09001460 }
1461
1462 ret = SN_ERR_INTERNAL;
1463 if (proto->connect)
Willy Tarreau02c62122014-12-08 12:11:28 +01001464 ret = proto->connect(conn, check->type, quickack ? 2 : 0);
Simon Hormanb00d17a2014-06-13 16:18:16 +09001465 conn->flags |= CO_FL_WAKE_DATA;
1466 if (s->check.send_proxy) {
1467 conn->send_proxy_ofs = 1;
1468 conn->flags |= CO_FL_SEND_PROXY;
1469 }
1470
1471 return ret;
1472}
1473
1474/*
Willy Tarreaubaaee002006-06-26 02:48:02 +02001475 * manages a server health-check. Returns
1476 * the time the task accepts to wait, or TIME_ETERNITY for infinity.
1477 */
Simon Horman63a4a822012-03-19 07:24:41 +09001478static struct task *process_chk(struct task *t)
Willy Tarreaubaaee002006-06-26 02:48:02 +02001479{
Simon Horman4a741432013-02-23 15:35:38 +09001480 struct check *check = t->context;
1481 struct server *s = check->server;
1482 struct connection *conn = check->conn;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001483 int rv;
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001484 int ret;
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001485 int expired = tick_is_expired(t->expire, now_ms);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001486
Willy Tarreau2c115e52013-12-11 19:41:16 +01001487 if (!(check->state & CHK_ST_INPROGRESS)) {
Willy Tarreau5a78f362012-11-23 12:47:05 +01001488 /* no check currently running */
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001489 if (!expired) /* woke up too early */
Willy Tarreau26c25062009-03-08 09:38:41 +01001490 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001491
Simon Horman671b6f02013-11-25 10:46:39 +09001492 /* we don't send any health-checks when the proxy is
1493 * stopped, the server should not be checked or the check
1494 * is disabled.
Willy Tarreaubaaee002006-06-26 02:48:02 +02001495 */
Willy Tarreau0d924cc2013-12-11 21:26:24 +01001496 if (((check->state & (CHK_ST_ENABLED | CHK_ST_PAUSED)) != CHK_ST_ENABLED) ||
Willy Tarreau33a08db2013-12-11 21:03:31 +01001497 s->proxy->state == PR_STSTOPPED)
Willy Tarreau5a78f362012-11-23 12:47:05 +01001498 goto reschedule;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001499
1500 /* we'll initiate a new check */
Simon Horman4a741432013-02-23 15:35:38 +09001501 set_server_check_status(check, HCHK_STATUS_START, NULL);
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001502
Willy Tarreau2c115e52013-12-11 19:41:16 +01001503 check->state |= CHK_ST_INPROGRESS;
Simon Horman4a741432013-02-23 15:35:38 +09001504 check->bi->p = check->bi->data;
1505 check->bi->i = 0;
1506 check->bo->p = check->bo->data;
1507 check->bo->o = 0;
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001508
Simon Hormanb00d17a2014-06-13 16:18:16 +09001509 ret = connect_chk(t);
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001510 switch (ret) {
Simon Hormanb00d17a2014-06-13 16:18:16 +09001511 case SN_ERR_UP:
1512 return t;
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001513 case SN_ERR_NONE:
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001514 /* we allow up to min(inter, timeout.connect) for a connection
1515 * to establish but only when timeout.check is set
1516 * as it may be to short for a full check otherwise
1517 */
Simon Horman4a741432013-02-23 15:35:38 +09001518 t->expire = tick_add(now_ms, MS_TO_TICKS(check->inter));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001519
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001520 if (s->proxy->timeout.check && s->proxy->timeout.connect) {
1521 int t_con = tick_add(now_ms, s->proxy->timeout.connect);
1522 t->expire = tick_first(t->expire, t_con);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001523 }
Willy Tarreau06559ac2013-12-05 01:53:08 +01001524
1525 if (check->type)
1526 conn_data_want_recv(conn); /* prepare for reading a possible reply */
1527
Willy Tarreau5a78f362012-11-23 12:47:05 +01001528 goto reschedule;
1529
1530 case SN_ERR_SRVTO: /* ETIMEDOUT */
1531 case SN_ERR_SRVCL: /* ECONNREFUSED, ENETUNREACH, ... */
Willy Tarreau4bd07de2014-01-24 16:10:57 +01001532 conn->flags |= CO_FL_ERROR;
1533 chk_report_conn_err(conn, errno, 0);
Willy Tarreau5a78f362012-11-23 12:47:05 +01001534 break;
1535 case SN_ERR_PRXCOND:
1536 case SN_ERR_RESOURCE:
1537 case SN_ERR_INTERNAL:
Willy Tarreau4bd07de2014-01-24 16:10:57 +01001538 conn->flags |= CO_FL_ERROR;
1539 chk_report_conn_err(conn, 0, 0);
Willy Tarreau5a78f362012-11-23 12:47:05 +01001540 break;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001541 }
1542
Willy Tarreau5a78f362012-11-23 12:47:05 +01001543 /* here, we have seen a synchronous error, no fd was allocated */
Willy Tarreau6b0a8502012-11-23 08:51:32 +01001544
Willy Tarreau2c115e52013-12-11 19:41:16 +01001545 check->state &= ~CHK_ST_INPROGRESS;
Willy Tarreau4eec5472014-05-20 22:32:27 +02001546 check_notify_failure(check);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001547
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001548 /* we allow up to min(inter, timeout.connect) for a connection
1549 * to establish but only when timeout.check is set
1550 * as it may be to short for a full check otherwise
1551 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001552 while (tick_is_expired(t->expire, now_ms)) {
1553 int t_con;
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001554
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001555 t_con = tick_add(t->expire, s->proxy->timeout.connect);
Simon Horman4a741432013-02-23 15:35:38 +09001556 t->expire = tick_add(t->expire, MS_TO_TICKS(check->inter));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001557
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001558 if (s->proxy->timeout.check)
1559 t->expire = tick_first(t->expire, t_con);
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001560 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001561 }
1562 else {
Willy Tarreauf1503172012-09-28 19:39:36 +02001563 /* there was a test running.
1564 * First, let's check whether there was an uncaught error,
1565 * which can happen on connect timeout or error.
1566 */
Simon Horman9ac7cab2014-06-20 12:29:47 +09001567 if (check->result == CHK_RES_UNKNOWN) {
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001568 /* good connection is enough for pure TCP check */
1569 if ((conn->flags & CO_FL_CONNECTED) && !check->type) {
Simon Horman4a741432013-02-23 15:35:38 +09001570 if (check->use_ssl)
1571 set_server_check_status(check, HCHK_STATUS_L6OK, NULL);
Willy Tarreauf1503172012-09-28 19:39:36 +02001572 else
Simon Horman4a741432013-02-23 15:35:38 +09001573 set_server_check_status(check, HCHK_STATUS_L4OK, NULL);
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001574 }
Willy Tarreau25e2ab52013-12-04 11:17:05 +01001575 else if ((conn->flags & CO_FL_ERROR) || expired) {
1576 chk_report_conn_err(conn, 0, expired);
Willy Tarreauf1503172012-09-28 19:39:36 +02001577 }
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001578 else
1579 goto out_wait; /* timeout not reached, wait again */
Willy Tarreauf1503172012-09-28 19:39:36 +02001580 }
1581
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001582 /* check complete or aborted */
Willy Tarreau5ba04f62013-02-12 15:23:12 +01001583 if (conn->xprt) {
1584 /* The check was aborted and the connection was not yet closed.
1585 * This can happen upon timeout, or when an external event such
1586 * as a failed response coupled with "observe layer7" caused the
1587 * server state to be suddenly changed.
1588 */
Willy Tarreau46be2e52014-01-20 12:10:52 +01001589 conn_drain(conn);
Willy Tarreauf79c8172013-10-21 16:30:56 +02001590 conn_force_close(conn);
Willy Tarreau5ba04f62013-02-12 15:23:12 +01001591 }
1592
Willy Tarreauaf549582014-05-16 17:37:50 +02001593 if (check->result == CHK_RES_FAILED) {
1594 /* a failure or timeout detected */
Willy Tarreau4eec5472014-05-20 22:32:27 +02001595 check_notify_failure(check);
Willy Tarreauaf549582014-05-16 17:37:50 +02001596 }
Willy Tarreaudb58b792014-05-21 13:57:23 +02001597 else if (check->result == CHK_RES_CONDPASS) {
1598 /* check is OK but asks for stopping mode */
1599 check_notify_stopping(check);
Willy Tarreauaf549582014-05-16 17:37:50 +02001600 }
Willy Tarreau3e048382014-05-21 10:30:54 +02001601 else if (check->result == CHK_RES_PASSED) {
1602 /* a success was detected */
1603 check_notify_success(check);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001604 }
Willy Tarreau2c115e52013-12-11 19:41:16 +01001605 check->state &= ~CHK_ST_INPROGRESS;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001606
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001607 rv = 0;
1608 if (global.spread_checks > 0) {
Simon Horman4a741432013-02-23 15:35:38 +09001609 rv = srv_getinter(check) * global.spread_checks / 100;
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001610 rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0)));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001611 }
Simon Horman4a741432013-02-23 15:35:38 +09001612 t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(check) + rv));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001613 }
Willy Tarreau5a78f362012-11-23 12:47:05 +01001614
1615 reschedule:
1616 while (tick_is_expired(t->expire, now_ms))
Simon Horman4a741432013-02-23 15:35:38 +09001617 t->expire = tick_add(t->expire, MS_TO_TICKS(check->inter));
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001618 out_wait:
Willy Tarreau26c25062009-03-08 09:38:41 +01001619 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001620}
1621
Simon Horman5c942422013-11-25 10:46:32 +09001622static int start_check_task(struct check *check, int mininter,
1623 int nbcheck, int srvpos)
1624{
1625 struct task *t;
1626 /* task for the check */
1627 if ((t = task_new()) == NULL) {
1628 Alert("Starting [%s:%s] check: out of memory.\n",
1629 check->server->proxy->id, check->server->id);
1630 return 0;
1631 }
1632
1633 check->task = t;
1634 t->process = process_chk;
1635 t->context = check;
1636
Willy Tarreau1746eec2014-04-25 10:46:47 +02001637 if (mininter < srv_getinter(check))
1638 mininter = srv_getinter(check);
1639
1640 if (global.max_spread_checks && mininter > global.max_spread_checks)
1641 mininter = global.max_spread_checks;
1642
Simon Horman5c942422013-11-25 10:46:32 +09001643 /* check this every ms */
Willy Tarreau1746eec2014-04-25 10:46:47 +02001644 t->expire = tick_add(now_ms, MS_TO_TICKS(mininter * srvpos / nbcheck));
Simon Horman5c942422013-11-25 10:46:32 +09001645 check->start = now;
1646 task_queue(t);
1647
1648 return 1;
1649}
1650
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001651/*
1652 * Start health-check.
1653 * Returns 0 if OK, -1 if error, and prints the error in this case.
1654 */
1655int start_checks() {
1656
1657 struct proxy *px;
1658 struct server *s;
1659 struct task *t;
Simon Horman4a741432013-02-23 15:35:38 +09001660 int nbcheck=0, mininter=0, srvpos=0;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001661
Willy Tarreau2c43a1e2007-10-14 23:05:39 +02001662 /* 1- count the checkers to run simultaneously.
1663 * We also determine the minimum interval among all of those which
1664 * have an interval larger than SRV_CHK_INTER_THRES. This interval
1665 * will be used to spread their start-up date. Those which have
Jamie Gloudon801a0a32012-08-25 00:18:33 -04001666 * a shorter interval will start independently and will not dictate
Willy Tarreau2c43a1e2007-10-14 23:05:39 +02001667 * too short an interval for all others.
1668 */
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001669 for (px = proxy; px; px = px->next) {
1670 for (s = px->srv; s; s = s->next) {
Willy Tarreaue7b73482013-11-21 11:50:50 +01001671 if (s->slowstart) {
1672 if ((t = task_new()) == NULL) {
1673 Alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
1674 return -1;
1675 }
1676 /* We need a warmup task that will be called when the server
1677 * state switches from down to up.
1678 */
1679 s->warmup = t;
1680 t->process = server_warmup;
1681 t->context = s;
1682 t->expire = TICK_ETERNITY;
1683 }
1684
Willy Tarreaud8514a22013-12-11 21:10:14 +01001685 if (s->check.state & CHK_ST_CONFIGURED) {
1686 nbcheck++;
1687 if ((srv_getinter(&s->check) >= SRV_CHK_INTER_THRES) &&
1688 (!mininter || mininter > srv_getinter(&s->check)))
1689 mininter = srv_getinter(&s->check);
1690 }
Willy Tarreau15f39102013-12-11 20:41:18 +01001691
Willy Tarreaud8514a22013-12-11 21:10:14 +01001692 if (s->agent.state & CHK_ST_CONFIGURED) {
1693 nbcheck++;
1694 if ((srv_getinter(&s->agent) >= SRV_CHK_INTER_THRES) &&
1695 (!mininter || mininter > srv_getinter(&s->agent)))
1696 mininter = srv_getinter(&s->agent);
1697 }
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001698 }
1699 }
1700
Simon Horman4a741432013-02-23 15:35:38 +09001701 if (!nbcheck)
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001702 return 0;
1703
1704 srand((unsigned)time(NULL));
1705
1706 /*
1707 * 2- start them as far as possible from each others. For this, we will
1708 * start them after their interval set to the min interval divided by
1709 * the number of servers, weighted by the server's position in the list.
1710 */
1711 for (px = proxy; px; px = px->next) {
1712 for (s = px->srv; s; s = s->next) {
Simon Hormand60d6912013-11-25 10:46:36 +09001713 /* A task for the main check */
Willy Tarreauff5ae352013-12-11 20:36:34 +01001714 if (s->check.state & CHK_ST_CONFIGURED) {
Simon Hormand60d6912013-11-25 10:46:36 +09001715 if (!start_check_task(&s->check, mininter, nbcheck, srvpos))
1716 return -1;
1717 srvpos++;
1718 }
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001719
Simon Hormand60d6912013-11-25 10:46:36 +09001720 /* A task for a auxiliary agent check */
Willy Tarreauff5ae352013-12-11 20:36:34 +01001721 if (s->agent.state & CHK_ST_CONFIGURED) {
Simon Hormand60d6912013-11-25 10:46:36 +09001722 if (!start_check_task(&s->agent, mininter, nbcheck, srvpos)) {
1723 return -1;
1724 }
1725 srvpos++;
1726 }
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001727 }
1728 }
1729 return 0;
1730}
Willy Tarreaubaaee002006-06-26 02:48:02 +02001731
1732/*
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001733 * Perform content verification check on data in s->check.buffer buffer.
Willy Tarreaubd741542010-03-16 18:46:54 +01001734 * The buffer MUST be terminated by a null byte before calling this function.
1735 * Sets server status appropriately. The caller is responsible for ensuring
1736 * that the buffer contains at least 13 characters. If <done> is zero, we may
1737 * return 0 to indicate that data is required to decide of a match.
1738 */
1739static int httpchk_expect(struct server *s, int done)
1740{
1741 static char status_msg[] = "HTTP status check returned code <000>";
1742 char status_code[] = "000";
1743 char *contentptr;
1744 int crlf;
1745 int ret;
1746
1747 switch (s->proxy->options2 & PR_O2_EXP_TYPE) {
1748 case PR_O2_EXP_STS:
1749 case PR_O2_EXP_RSTS:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001750 memcpy(status_code, s->check.bi->data + 9, 3);
1751 memcpy(status_msg + strlen(status_msg) - 4, s->check.bi->data + 9, 3);
Willy Tarreaubd741542010-03-16 18:46:54 +01001752
1753 if ((s->proxy->options2 & PR_O2_EXP_TYPE) == PR_O2_EXP_STS)
1754 ret = strncmp(s->proxy->expect_str, status_code, 3) == 0;
1755 else
Thierry FOURNIER09af0d62014-06-18 11:35:54 +02001756 ret = regex_exec(s->proxy->expect_regex, status_code);
Willy Tarreaubd741542010-03-16 18:46:54 +01001757
1758 /* we necessarily have the response, so there are no partial failures */
1759 if (s->proxy->options2 & PR_O2_EXP_INV)
1760 ret = !ret;
1761
Simon Horman4a741432013-02-23 15:35:38 +09001762 set_server_check_status(&s->check, ret ? HCHK_STATUS_L7OKD : HCHK_STATUS_L7STS, status_msg);
Willy Tarreaubd741542010-03-16 18:46:54 +01001763 break;
1764
1765 case PR_O2_EXP_STR:
1766 case PR_O2_EXP_RSTR:
1767 /* very simple response parser: ignore CR and only count consecutive LFs,
1768 * stop with contentptr pointing to first char after the double CRLF or
1769 * to '\0' if crlf < 2.
1770 */
1771 crlf = 0;
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001772 for (contentptr = s->check.bi->data; *contentptr; contentptr++) {
Willy Tarreaubd741542010-03-16 18:46:54 +01001773 if (crlf >= 2)
1774 break;
1775 if (*contentptr == '\r')
1776 continue;
1777 else if (*contentptr == '\n')
1778 crlf++;
1779 else
1780 crlf = 0;
1781 }
1782
1783 /* Check that response contains a body... */
1784 if (crlf < 2) {
1785 if (!done)
1786 return 0;
1787
Simon Horman4a741432013-02-23 15:35:38 +09001788 set_server_check_status(&s->check, HCHK_STATUS_L7RSP,
Willy Tarreaubd741542010-03-16 18:46:54 +01001789 "HTTP content check could not find a response body");
1790 return 1;
1791 }
1792
1793 /* Check that response body is not empty... */
1794 if (*contentptr == '\0') {
Willy Tarreaua164fb52011-04-13 09:32:41 +02001795 if (!done)
1796 return 0;
1797
Simon Horman4a741432013-02-23 15:35:38 +09001798 set_server_check_status(&s->check, HCHK_STATUS_L7RSP,
Willy Tarreaubd741542010-03-16 18:46:54 +01001799 "HTTP content check found empty response body");
1800 return 1;
1801 }
1802
1803 /* Check the response content against the supplied string