blob: d430c9b903acbdf4bdd61eecf0a4403517675dde [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
2 * Health-checks functions.
3 *
Willy Tarreau26c25062009-03-08 09:38:41 +01004 * Copyright 2000-2009 Willy Tarreau <w@1wt.eu>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02005 * Copyright 2007-2009 Krzysztof Piotr Oledzki <ole@ans.pl>
Willy Tarreaubaaee002006-06-26 02:48:02 +02006 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
11 *
12 */
13
Willy Tarreaub8816082008-01-18 12:18:15 +010014#include <assert.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020015#include <ctype.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020016#include <errno.h>
17#include <fcntl.h>
18#include <stdio.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020019#include <stdlib.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020020#include <string.h>
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +020021#include <time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020022#include <unistd.h>
23#include <sys/socket.h>
Dmitry Sivachenkocaf58982009-08-24 15:11:06 +040024#include <sys/types.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020025#include <netinet/in.h>
Willy Tarreau1274bc42009-07-15 07:16:31 +020026#include <netinet/tcp.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020027#include <arpa/inet.h>
28
Willy Tarreauc7e42382012-08-24 19:22:53 +020029#include <common/chunk.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020030#include <common/compat.h>
31#include <common/config.h>
32#include <common/mini-clist.h>
Willy Tarreau83749182007-04-15 20:56:27 +020033#include <common/standard.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020034#include <common/time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020035
36#include <types/global.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020037
38#include <proto/backend.h>
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020039#include <proto/checks.h>
Simon Hormana2b9dad2013-02-12 10:45:54 +090040#include <proto/dumpstats.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020041#include <proto/fd.h>
42#include <proto/log.h>
43#include <proto/queue.h>
Willy Tarreauc6f4ce82009-06-10 11:09:37 +020044#include <proto/port_range.h>
Willy Tarreau3d300592007-03-18 18:34:41 +010045#include <proto/proto_http.h>
Willy Tarreaue8c66af2008-01-13 18:40:14 +010046#include <proto/proto_tcp.h>
Willy Tarreau2b5652f2006-12-31 17:46:05 +010047#include <proto/proxy.h>
Willy Tarreaufb56aab2012-09-28 14:40:02 +020048#include <proto/raw_sock.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020049#include <proto/server.h>
Simon Hormane0d1bfb2011-06-21 14:34:58 +090050#include <proto/session.h>
Willy Tarreau9e000c62011-03-10 14:03:36 +010051#include <proto/stream_interface.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020052#include <proto/task.h>
53
Willy Tarreaubd741542010-03-16 18:46:54 +010054static int httpchk_expect(struct server *s, int done);
55
Simon Horman63a4a822012-03-19 07:24:41 +090056static const struct check_status check_statuses[HCHK_STATUS_SIZE] = {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020057 [HCHK_STATUS_UNKNOWN] = { SRV_CHK_UNKNOWN, "UNK", "Unknown" },
58 [HCHK_STATUS_INI] = { SRV_CHK_UNKNOWN, "INI", "Initializing" },
59 [HCHK_STATUS_START] = { /* SPECIAL STATUS*/ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020060
Willy Tarreaud3aac702012-11-23 11:32:12 +010061 [HCHK_STATUS_HANA] = { SRV_CHK_FAILED, "HANA", "Health analyze" },
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010062
Willy Tarreaud3aac702012-11-23 11:32:12 +010063 [HCHK_STATUS_SOCKERR] = { SRV_CHK_FAILED, "SOCKERR", "Socket error" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020064
Willy Tarreaud3aac702012-11-23 11:32:12 +010065 [HCHK_STATUS_L4OK] = { SRV_CHK_PASSED, "L4OK", "Layer4 check passed" },
66 [HCHK_STATUS_L4TOUT] = { SRV_CHK_FAILED, "L4TOUT", "Layer4 timeout" },
67 [HCHK_STATUS_L4CON] = { SRV_CHK_FAILED, "L4CON", "Layer4 connection problem" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020068
Willy Tarreaud3aac702012-11-23 11:32:12 +010069 [HCHK_STATUS_L6OK] = { SRV_CHK_PASSED, "L6OK", "Layer6 check passed" },
70 [HCHK_STATUS_L6TOUT] = { SRV_CHK_FAILED, "L6TOUT", "Layer6 timeout" },
71 [HCHK_STATUS_L6RSP] = { SRV_CHK_FAILED, "L6RSP", "Layer6 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020072
Willy Tarreaud3aac702012-11-23 11:32:12 +010073 [HCHK_STATUS_L7TOUT] = { SRV_CHK_FAILED, "L7TOUT", "Layer7 timeout" },
74 [HCHK_STATUS_L7RSP] = { SRV_CHK_FAILED, "L7RSP", "Layer7 invalid response" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020075
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +020076 [HCHK_STATUS_L57DATA] = { /* DUMMY STATUS */ },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020077
Willy Tarreaud3aac702012-11-23 11:32:12 +010078 [HCHK_STATUS_L7OKD] = { SRV_CHK_PASSED, "L7OK", "Layer7 check passed" },
79 [HCHK_STATUS_L7OKCD] = { SRV_CHK_PASSED | SRV_CHK_DISABLE, "L7OKC", "Layer7 check conditionally passed" },
80 [HCHK_STATUS_L7STS] = { SRV_CHK_FAILED, "L7STS", "Layer7 wrong status" },
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020081};
82
Simon Horman63a4a822012-03-19 07:24:41 +090083static const struct analyze_status analyze_statuses[HANA_STATUS_SIZE] = { /* 0: ignore, 1: error, 2: OK */
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +010084 [HANA_STATUS_UNKNOWN] = { "Unknown", { 0, 0 }},
85
86 [HANA_STATUS_L4_OK] = { "L4 successful connection", { 2, 0 }},
87 [HANA_STATUS_L4_ERR] = { "L4 unsuccessful connection", { 1, 1 }},
88
89 [HANA_STATUS_HTTP_OK] = { "Correct http response", { 0, 2 }},
90 [HANA_STATUS_HTTP_STS] = { "Wrong http response", { 0, 1 }},
91 [HANA_STATUS_HTTP_HDRRSP] = { "Invalid http response (headers)", { 0, 1 }},
92 [HANA_STATUS_HTTP_RSP] = { "Invalid http response", { 0, 1 }},
93
94 [HANA_STATUS_HTTP_READ_ERROR] = { "Read error (http)", { 0, 1 }},
95 [HANA_STATUS_HTTP_READ_TIMEOUT] = { "Read timeout (http)", { 0, 1 }},
96 [HANA_STATUS_HTTP_BROKEN_PIPE] = { "Close from server (http)", { 0, 1 }},
97};
98
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +020099/*
100 * Convert check_status code to description
101 */
102const char *get_check_status_description(short check_status) {
103
104 const char *desc;
105
106 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200107 desc = check_statuses[check_status].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200108 else
109 desc = NULL;
110
111 if (desc && *desc)
112 return desc;
113 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200114 return check_statuses[HCHK_STATUS_UNKNOWN].desc;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200115}
116
117/*
118 * Convert check_status code to short info
119 */
120const char *get_check_status_info(short check_status) {
121
122 const char *info;
123
124 if (check_status < HCHK_STATUS_SIZE)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200125 info = check_statuses[check_status].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200126 else
127 info = NULL;
128
129 if (info && *info)
130 return info;
131 else
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200132 return check_statuses[HCHK_STATUS_UNKNOWN].info;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200133}
134
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100135const char *get_analyze_status(short analyze_status) {
136
137 const char *desc;
138
139 if (analyze_status < HANA_STATUS_SIZE)
140 desc = analyze_statuses[analyze_status].desc;
141 else
142 desc = NULL;
143
144 if (desc && *desc)
145 return desc;
146 else
147 return analyze_statuses[HANA_STATUS_UNKNOWN].desc;
148}
149
Simon Horman4a741432013-02-23 15:35:38 +0900150static void server_status_printf(struct chunk *msg, struct server *s, struct check *check, int xferred) {
Willy Tarreau44267702011-10-28 15:35:33 +0200151 if (s->track)
Willy Tarreau77804732012-10-29 16:14:26 +0100152 chunk_appendf(msg, " via %s/%s",
Willy Tarreau44267702011-10-28 15:35:33 +0200153 s->track->proxy->id, s->track->id);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200154
Simon Horman4a741432013-02-23 15:35:38 +0900155 if (check) {
156 chunk_appendf(msg, ", reason: %s", get_check_status_description(check->status));
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200157
Simon Horman4a741432013-02-23 15:35:38 +0900158 if (check->status >= HCHK_STATUS_L57DATA)
159 chunk_appendf(msg, ", code: %d", check->code);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200160
Simon Horman4a741432013-02-23 15:35:38 +0900161 if (*check->desc) {
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200162 struct chunk src;
163
Willy Tarreau77804732012-10-29 16:14:26 +0100164 chunk_appendf(msg, ", info: \"");
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200165
Simon Horman4a741432013-02-23 15:35:38 +0900166 chunk_initlen(&src, check->desc, 0, strlen(check->desc));
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200167 chunk_asciiencode(msg, &src, '"');
168
Willy Tarreau77804732012-10-29 16:14:26 +0100169 chunk_appendf(msg, "\"");
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200170 }
171
Simon Horman4a741432013-02-23 15:35:38 +0900172 if (check->duration >= 0)
173 chunk_appendf(msg, ", check duration: %ldms", check->duration);
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200174 }
175
Krzysztof Piotr Oledzki3bb05712010-09-27 13:10:50 +0200176 if (xferred >= 0) {
Krzysztof Piotr Oledzkib16a6072010-01-10 21:12:58 +0100177 if (!(s->state & SRV_RUNNING))
Willy Tarreau77804732012-10-29 16:14:26 +0100178 chunk_appendf(msg, ". %d active and %d backup servers left.%s"
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100179 " %d sessions active, %d requeued, %d remaining in queue",
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200180 s->proxy->srv_act, s->proxy->srv_bck,
181 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
182 s->cur_sess, xferred, s->nbpend);
183 else
Willy Tarreau77804732012-10-29 16:14:26 +0100184 chunk_appendf(msg, ". %d active and %d backup servers online.%s"
Krzysztof Piotr Oledzki9f2b9d52010-01-11 13:16:27 +0100185 " %d sessions requeued, %d total in queue",
Krzysztof Piotr Oledzki99ab5f82009-09-27 17:28:21 +0200186 s->proxy->srv_act, s->proxy->srv_bck,
187 (s->proxy->srv_bck && !s->proxy->srv_act) ? " Running on backup." : "",
188 xferred, s->nbpend);
189 }
190}
191
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200192/*
Simon Horman4a741432013-02-23 15:35:38 +0900193 * Set check->status, update check->duration and fill check->result with
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200194 * an adequate SRV_CHK_* value.
195 *
196 * Show information in logs about failed health check if server is UP
197 * or succeeded health checks if server is DOWN.
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200198 */
Simon Horman4a741432013-02-23 15:35:38 +0900199static void set_server_check_status(struct check *check, short status, const char *desc)
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100200{
Simon Horman4a741432013-02-23 15:35:38 +0900201 struct server *s = check->server;
202
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200203 if (status == HCHK_STATUS_START) {
Simon Horman4a741432013-02-23 15:35:38 +0900204 check->result = SRV_CHK_UNKNOWN; /* no result yet */
205 check->desc[0] = '\0';
206 check->start = now;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200207 return;
208 }
209
Simon Horman4a741432013-02-23 15:35:38 +0900210 if (!check->status)
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200211 return;
212
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200213 if (desc && *desc) {
Simon Horman4a741432013-02-23 15:35:38 +0900214 strncpy(check->desc, desc, HCHK_DESC_LEN-1);
215 check->desc[HCHK_DESC_LEN-1] = '\0';
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200216 } else
Simon Horman4a741432013-02-23 15:35:38 +0900217 check->desc[0] = '\0';
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200218
Simon Horman4a741432013-02-23 15:35:38 +0900219 check->status = status;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200220 if (check_statuses[status].result)
Simon Horman4a741432013-02-23 15:35:38 +0900221 check->result = check_statuses[status].result;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200222
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100223 if (status == HCHK_STATUS_HANA)
Simon Horman4a741432013-02-23 15:35:38 +0900224 check->duration = -1;
225 else if (!tv_iszero(&check->start)) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200226 /* set_server_check_status() may be called more than once */
Simon Horman4a741432013-02-23 15:35:38 +0900227 check->duration = tv_ms_elapsed(&check->start, &now);
228 tv_zero(&check->start);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200229 }
230
Simon Horman2f1f9552013-11-25 10:46:37 +0900231 /* Failure to connect to the agent as a secondary check should not
232 * cause the server to be marked down. So only log status changes
233 * for HCHK_STATUS_* statuses */
234 if (check == &s->agent && check->status < HCHK_STATUS_L7TOUT)
235 return;
236
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200237 if (s->proxy->options2 & PR_O2_LOGHCHKS &&
Simon Horman125d0992013-02-24 17:23:38 +0900238 (((check->health != 0) && (check->result & SRV_CHK_FAILED)) ||
Simon Horman8c3d0be2013-11-25 10:46:40 +0900239 (((check->health != check->rise + check->fall - 1) ||
240 (!s->uweight && !(s->state & SRV_DRAIN)) ||
241 (s->uweight && (s->state & SRV_DRAIN))) &&
242 (check->result & SRV_CHK_PASSED)) ||
Simon Horman4a741432013-02-23 15:35:38 +0900243 ((s->state & SRV_GOINGDOWN) && !(check->result & SRV_CHK_DISABLE)) ||
244 (!(s->state & SRV_GOINGDOWN) && (check->result & SRV_CHK_DISABLE)))) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200245
246 int health, rise, fall, state;
247
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100248 chunk_reset(&trash);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200249
250 /* FIXME begin: calculate local version of the health/rise/fall/state */
Simon Horman125d0992013-02-24 17:23:38 +0900251 health = check->health;
Simon Horman58c32972013-11-25 10:46:38 +0900252 rise = check->rise;
253 fall = check->fall;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200254 state = s->state;
255
Simon Horman4a741432013-02-23 15:35:38 +0900256 if (check->result & SRV_CHK_FAILED) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200257 if (health > rise) {
258 health--; /* still good */
259 } else {
260 if (health == rise)
261 state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
262
263 health = 0;
264 }
265 }
266
Simon Horman4a741432013-02-23 15:35:38 +0900267 if (check->result & SRV_CHK_PASSED) {
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200268 if (health < rise + fall - 1) {
269 health++; /* was bad, stays for a while */
270
271 if (health == rise)
272 state |= SRV_RUNNING;
273
274 if (health >= rise)
275 health = rise + fall - 1; /* OK now */
276 }
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100277
278 /* clear consecutive_errors if observing is enabled */
279 if (s->onerror)
280 s->consecutive_errors = 0;
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200281 }
282 /* FIXME end: calculate local version of the health/rise/fall/state */
283
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100284 chunk_appendf(&trash,
285 "Health check for %sserver %s/%s %s%s",
286 s->state & SRV_BACKUP ? "backup " : "",
287 s->proxy->id, s->id,
Simon Horman4a741432013-02-23 15:35:38 +0900288 (check->result & SRV_CHK_DISABLE)?"conditionally ":"",
289 (check->result & SRV_CHK_PASSED)?"succeeded":"failed");
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200290
Simon Horman4a741432013-02-23 15:35:38 +0900291 server_status_printf(&trash, s, check, -1);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200292
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100293 chunk_appendf(&trash, ", status: %d/%d %s",
294 (state & SRV_RUNNING) ? (health - rise + 1) : (health),
295 (state & SRV_RUNNING) ? (fall) : (rise),
Simon Horman8c3d0be2013-11-25 10:46:40 +0900296 (state & SRV_RUNNING)?(s->eweight?"UP":"DRAIN"):"DOWN");
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200297
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100298 Warning("%s.\n", trash.str);
299 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200300 }
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200301}
302
Willy Tarreau48494c02007-11-30 10:41:39 +0100303/* sends a log message when a backend goes down, and also sets last
304 * change date.
305 */
306static void set_backend_down(struct proxy *be)
307{
308 be->last_change = now.tv_sec;
309 be->down_trans++;
310
311 Alert("%s '%s' has no server available!\n", proxy_type_str(be), be->id);
312 send_log(be, LOG_EMERG, "%s %s has no server available!\n", proxy_type_str(be), be->id);
313}
314
315/* Redistribute pending connections when a server goes down. The number of
316 * connections redistributed is returned.
317 */
318static int redistribute_pending(struct server *s)
319{
320 struct pendconn *pc, *pc_bck, *pc_end;
321 int xferred = 0;
322
323 FOREACH_ITEM_SAFE(pc, pc_bck, &s->pendconns, pc_end, struct pendconn *, list) {
324 struct session *sess = pc->sess;
Willy Tarreau4de91492010-01-22 19:10:05 +0100325 if ((sess->be->options & (PR_O_REDISP|PR_O_PERSIST)) == PR_O_REDISP &&
326 !(sess->flags & SN_FORCE_PRST)) {
Willy Tarreau48494c02007-11-30 10:41:39 +0100327 /* The REDISP option was specified. We will ignore
328 * cookie and force to balance or use the dispatcher.
329 */
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100330
Krzysztof Piotr Oledzki5a329cf2008-02-22 03:50:19 +0100331 /* it's left to the dispatcher to choose a server */
Willy Tarreau48494c02007-11-30 10:41:39 +0100332 sess->flags &= ~(SN_DIRECT | SN_ASSIGNED | SN_ADDR_SET);
Krzysztof Piotr Oledzki25b501a2008-01-06 16:36:16 +0100333
Willy Tarreau48494c02007-11-30 10:41:39 +0100334 pendconn_free(pc);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200335 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100336 xferred++;
337 }
338 }
339 return xferred;
340}
341
342/* Check for pending connections at the backend, and assign some of them to
343 * the server coming up. The server's weight is checked before being assigned
344 * connections it may not be able to handle. The total number of transferred
345 * connections is returned.
346 */
347static int check_for_pending(struct server *s)
348{
349 int xferred;
350
351 if (!s->eweight)
352 return 0;
353
354 for (xferred = 0; !s->maxconn || xferred < srv_dynamic_maxconn(s); xferred++) {
355 struct session *sess;
356 struct pendconn *p;
357
358 p = pendconn_from_px(s->proxy);
359 if (!p)
360 break;
Willy Tarreau3fdb3662012-11-12 00:42:33 +0100361 p->sess->target = &s->obj_type;
Willy Tarreau48494c02007-11-30 10:41:39 +0100362 sess = p->sess;
363 pendconn_free(p);
Willy Tarreaufdccded2008-08-29 18:19:04 +0200364 task_wakeup(sess->task, TASK_WOKEN_RES);
Willy Tarreau48494c02007-11-30 10:41:39 +0100365 }
366 return xferred;
367}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200368
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700369/* Shutdown all connections of a server. The caller must pass a termination
370 * code in <why>, which must be one of SN_ERR_* indicating the reason for the
371 * shutdown.
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900372 */
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700373static void shutdown_sessions(struct server *srv, int why)
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900374{
375 struct session *session, *session_bck;
376
Willy Tarreaua2a64e92011-09-07 23:01:56 +0200377 list_for_each_entry_safe(session, session_bck, &srv->actconns, by_srv)
378 if (session->srv_conn == srv)
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700379 session_shutdown(session, why);
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900380}
381
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700382/* Shutdown all connections of all backup servers of a proxy. The caller must
383 * pass a termination code in <why>, which must be one of SN_ERR_* indicating
384 * the reason for the shutdown.
385 */
386static void shutdown_backup_sessions(struct proxy *px, int why)
387{
388 struct server *srv;
389
390 for (srv = px->srv; srv != NULL; srv = srv->next)
391 if (srv->state & SRV_BACKUP)
392 shutdown_sessions(srv, why);
393}
394
Willy Tarreaubaaee002006-06-26 02:48:02 +0200395/* Sets server <s> down, notifies by all available means, recounts the
396 * remaining servers on the proxy and transfers queued sessions whenever
Willy Tarreau5af3a692007-07-24 23:32:33 +0200397 * possible to other servers. It automatically recomputes the number of
398 * servers, but not the map.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200399 */
Simon Horman4a741432013-02-23 15:35:38 +0900400void set_server_down(struct check *check)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200401{
Simon Horman4a741432013-02-23 15:35:38 +0900402 struct server *s = check->server;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100403 struct server *srv;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200404 int xferred;
405
Cyril Bontécd19e512010-01-31 22:34:03 +0100406 if (s->state & SRV_MAINTAIN) {
Simon Horman58c32972013-11-25 10:46:38 +0900407 check->health = check->rise;
Cyril Bontécd19e512010-01-31 22:34:03 +0100408 }
409
Simon Horman58c32972013-11-25 10:46:38 +0900410 if ((s->state & SRV_RUNNING && check->health == check->rise) || s->track) {
Willy Tarreau48494c02007-11-30 10:41:39 +0100411 int srv_was_paused = s->state & SRV_GOINGDOWN;
Willy Tarreaud64d2252010-10-17 17:16:42 +0200412 int prev_srv_count = s->proxy->srv_bck + s->proxy->srv_act;
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200413
414 s->last_change = now.tv_sec;
Willy Tarreau48494c02007-11-30 10:41:39 +0100415 s->state &= ~(SRV_RUNNING | SRV_GOINGDOWN);
Willy Tarreau9580d162012-05-19 19:07:40 +0200416 if (s->proxy->lbprm.set_server_status_down)
417 s->proxy->lbprm.set_server_status_down(s);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200418
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900419 if (s->onmarkeddown & HANA_ONMARKEDDOWN_SHUTDOWNSESSIONS)
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700420 shutdown_sessions(s, SN_ERR_DOWN);
Simon Hormane0d1bfb2011-06-21 14:34:58 +0900421
Willy Tarreaubaaee002006-06-26 02:48:02 +0200422 /* we might have sessions queued on this server and waiting for
423 * a connection. Those which are redispatchable will be queued
424 * to another server or to the proxy itself.
425 */
Willy Tarreau48494c02007-11-30 10:41:39 +0100426 xferred = redistribute_pending(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100427
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100428 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100429
Cyril Bontécd19e512010-01-31 22:34:03 +0100430 if (s->state & SRV_MAINTAIN) {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100431 chunk_appendf(&trash,
432 "%sServer %s/%s is DOWN for maintenance", s->state & SRV_BACKUP ? "Backup " : "",
433 s->proxy->id, s->id);
Cyril Bontécd19e512010-01-31 22:34:03 +0100434 } else {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100435 chunk_appendf(&trash,
436 "%sServer %s/%s is DOWN", s->state & SRV_BACKUP ? "Backup " : "",
437 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100438
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100439 server_status_printf(&trash, s,
Simon Horman4a741432013-02-23 15:35:38 +0900440 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? check : 0),
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100441 xferred);
Cyril Bontécd19e512010-01-31 22:34:03 +0100442 }
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100443 Warning("%s.\n", trash.str);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200444
Willy Tarreau48494c02007-11-30 10:41:39 +0100445 /* we don't send an alert if the server was previously paused */
446 if (srv_was_paused)
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100447 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Willy Tarreau48494c02007-11-30 10:41:39 +0100448 else
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100449 send_log(s->proxy, LOG_ALERT, "%s.\n", trash.str);
Krzysztof Oledzki85130942007-10-22 16:21:10 +0200450
Willy Tarreaud64d2252010-10-17 17:16:42 +0200451 if (prev_srv_count && s->proxy->srv_bck == 0 && s->proxy->srv_act == 0)
Willy Tarreau48494c02007-11-30 10:41:39 +0100452 set_backend_down(s->proxy);
453
Krzysztof Piotr Oledzki052d4fd2009-10-04 14:52:57 +0200454 s->counters.down_trans++;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100455
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100456 if (s->state & SRV_CHECKED)
Willy Tarreaubc16cd82013-11-27 16:52:23 +0100457 for (srv = s->tracknext; srv; srv = srv->tracknext)
458 if (!(srv->state & SRV_MAINTAIN))
Cyril Bontécd19e512010-01-31 22:34:03 +0100459 /* Only notify tracking servers that are not already in maintenance. */
Willy Tarreaubc16cd82013-11-27 16:52:23 +0100460 set_server_down(&srv->check);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200461 }
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100462
Simon Horman125d0992013-02-24 17:23:38 +0900463 check->health = 0; /* failure */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200464}
465
Simon Horman4a741432013-02-23 15:35:38 +0900466void set_server_up(struct check *check) {
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100467
Simon Horman4a741432013-02-23 15:35:38 +0900468 struct server *s = check->server;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100469 struct server *srv;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100470 int xferred;
Willy Tarreau45446782012-03-09 17:16:09 +0100471 unsigned int old_state = s->state;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100472
Cyril Bontécd19e512010-01-31 22:34:03 +0100473 if (s->state & SRV_MAINTAIN) {
Simon Horman58c32972013-11-25 10:46:38 +0900474 check->health = check->rise;
Cyril Bontécd19e512010-01-31 22:34:03 +0100475 }
476
Simon Horman58c32972013-11-25 10:46:38 +0900477 if ((s->check.health >= s->check.rise && s->agent.health >= s->agent.rise &&
478 check->health == check->rise) || s->track) {
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100479 if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0) {
480 if (s->proxy->last_change < now.tv_sec) // ignore negative times
481 s->proxy->down_time += now.tv_sec - s->proxy->last_change;
482 s->proxy->last_change = now.tv_sec;
483 }
484
485 if (s->last_change < now.tv_sec) // ignore negative times
486 s->down_time += now.tv_sec - s->last_change;
487
488 s->last_change = now.tv_sec;
489 s->state |= SRV_RUNNING;
Willy Tarreau45446782012-03-09 17:16:09 +0100490 s->state &= ~SRV_MAINTAIN;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100491
492 if (s->slowstart > 0) {
493 s->state |= SRV_WARMINGUP;
Willy Tarreau2e993902011-10-31 11:53:20 +0100494 task_schedule(s->warmup, tick_add(now_ms, MS_TO_TICKS(MAX(1000, s->slowstart / 20))));
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100495 }
Willy Tarreau004e0452013-11-21 11:22:01 +0100496
497 server_recalc_eweight(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100498
Justin Karnegeseb2c24a2012-05-24 15:28:52 -0700499 /* If the server is set with "on-marked-up shutdown-backup-sessions",
500 * and it's not a backup server and its effective weight is > 0,
501 * then it can accept new connections, so we shut down all sessions
502 * on all backup servers.
503 */
504 if ((s->onmarkedup & HANA_ONMARKEDUP_SHUTDOWNBACKUPSESSIONS) &&
505 !(s->state & SRV_BACKUP) && s->eweight)
506 shutdown_backup_sessions(s->proxy, SN_ERR_UP);
507
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100508 /* check if we can handle some connections queued at the proxy. We
509 * will take as many as we can handle.
510 */
511 xferred = check_for_pending(s);
512
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100513 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100514
Willy Tarreau45446782012-03-09 17:16:09 +0100515 if (old_state & SRV_MAINTAIN) {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100516 chunk_appendf(&trash,
517 "%sServer %s/%s is UP (leaving maintenance)", s->state & SRV_BACKUP ? "Backup " : "",
518 s->proxy->id, s->id);
Cyril Bontécd19e512010-01-31 22:34:03 +0100519 } else {
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100520 chunk_appendf(&trash,
521 "%sServer %s/%s is UP", s->state & SRV_BACKUP ? "Backup " : "",
522 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100523
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100524 server_status_printf(&trash, s,
Simon Horman4a741432013-02-23 15:35:38 +0900525 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? check : NULL),
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100526 xferred);
Cyril Bontécd19e512010-01-31 22:34:03 +0100527 }
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100528
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100529 Warning("%s.\n", trash.str);
530 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100531
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100532 if (s->state & SRV_CHECKED)
Willy Tarreaubc16cd82013-11-27 16:52:23 +0100533 for (srv = s->tracknext; srv; srv = srv->tracknext)
534 if (!(srv->state & SRV_MAINTAIN))
Cyril Bontécd19e512010-01-31 22:34:03 +0100535 /* Only notify tracking servers if they're not in maintenance. */
Willy Tarreaubc16cd82013-11-27 16:52:23 +0100536 set_server_up(&srv->check);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100537 }
538
Simon Horman58c32972013-11-25 10:46:38 +0900539 if (check->health >= check->rise)
540 check->health = check->rise + check->fall - 1; /* OK now */
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100541
542}
543
Simon Horman4a741432013-02-23 15:35:38 +0900544static void set_server_disabled(struct check *check) {
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100545
Simon Horman4a741432013-02-23 15:35:38 +0900546 struct server *s = check->server;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100547 struct server *srv;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100548 int xferred;
549
550 s->state |= SRV_GOINGDOWN;
Willy Tarreau9580d162012-05-19 19:07:40 +0200551 if (s->proxy->lbprm.set_server_status_down)
552 s->proxy->lbprm.set_server_status_down(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100553
554 /* we might have sessions queued on this server and waiting for
555 * a connection. Those which are redispatchable will be queued
556 * to another server or to the proxy itself.
557 */
558 xferred = redistribute_pending(s);
559
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100560 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100561
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100562 chunk_appendf(&trash,
563 "Load-balancing on %sServer %s/%s is disabled",
564 s->state & SRV_BACKUP ? "Backup " : "",
565 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100566
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100567 server_status_printf(&trash, s,
Simon Horman4a741432013-02-23 15:35:38 +0900568 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? check : NULL),
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100569 xferred);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100570
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100571 Warning("%s.\n", trash.str);
572 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100573
574 if (!s->proxy->srv_bck && !s->proxy->srv_act)
575 set_backend_down(s->proxy);
576
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100577 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100578 for(srv = s->tracknext; srv; srv = srv->tracknext)
Willy Tarreau81cf08c2013-11-28 11:27:16 +0100579 set_server_disabled(&srv->check);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100580}
581
Simon Horman4a741432013-02-23 15:35:38 +0900582static void set_server_enabled(struct check *check) {
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100583
Simon Horman4a741432013-02-23 15:35:38 +0900584 struct server *s = check->server;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100585 struct server *srv;
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100586 int xferred;
587
588 s->state &= ~SRV_GOINGDOWN;
Willy Tarreau9580d162012-05-19 19:07:40 +0200589 if (s->proxy->lbprm.set_server_status_up)
590 s->proxy->lbprm.set_server_status_up(s);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100591
592 /* check if we can handle some connections queued at the proxy. We
593 * will take as many as we can handle.
594 */
595 xferred = check_for_pending(s);
596
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100597 chunk_reset(&trash);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100598
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100599 chunk_appendf(&trash,
600 "Load-balancing on %sServer %s/%s is enabled again",
601 s->state & SRV_BACKUP ? "Backup " : "",
602 s->proxy->id, s->id);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100603
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100604 server_status_printf(&trash, s,
Simon Horman4a741432013-02-23 15:35:38 +0900605 ((!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) ? check : NULL),
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100606 xferred);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100607
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100608 Warning("%s.\n", trash.str);
609 send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.str);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100610
Krzysztof Piotr Oledzkif39c71c2009-01-30 00:52:49 +0100611 if (s->state & SRV_CHECKED)
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100612 for(srv = s->tracknext; srv; srv = srv->tracknext)
Willy Tarreau81cf08c2013-11-28 11:27:16 +0100613 set_server_enabled(&srv->check);
Krzysztof Piotr Oledzkic8b16fc2008-02-18 01:26:35 +0100614}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200615
Simon Hormand8583062013-11-25 10:46:33 +0900616static void check_failed(struct check *check)
617{
618 struct server *s = check->server;
619
Simon Horman2f1f9552013-11-25 10:46:37 +0900620 /* The agent secondary check should only cause a server to be marked
621 * as down if check->status is HCHK_STATUS_L7STS, which indicates
622 * that the agent returned "fail", "stopped" or "down".
623 * The implication here is that failure to connect to the agent
624 * as a secondary check should not cause the server to be marked
625 * down. */
626 if (check == &s->agent && check->status != HCHK_STATUS_L7STS)
627 return;
628
Simon Horman58c32972013-11-25 10:46:38 +0900629 if (check->health > check->rise) {
Simon Hormand8583062013-11-25 10:46:33 +0900630 check->health--; /* still good */
631 s->counters.failed_checks++;
632 }
633 else
634 set_server_down(check);
635}
636
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100637void health_adjust(struct server *s, short status)
638{
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100639 int failed;
640 int expire;
641
642 /* return now if observing nor health check is not enabled */
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200643 if (!s->observe || !s->check.task)
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100644 return;
645
646 if (s->observe >= HANA_OBS_SIZE)
647 return;
648
Willy Tarreaubb956662013-01-24 00:37:39 +0100649 if (status >= HANA_STATUS_SIZE || !analyze_statuses[status].desc)
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100650 return;
651
652 switch (analyze_statuses[status].lr[s->observe - 1]) {
653 case 1:
654 failed = 1;
655 break;
656
657 case 2:
658 failed = 0;
659 break;
660
661 default:
662 return;
663 }
664
665 if (!failed) {
666 /* good: clear consecutive_errors */
667 s->consecutive_errors = 0;
668 return;
669 }
670
671 s->consecutive_errors++;
672
673 if (s->consecutive_errors < s->consecutive_errors_limit)
674 return;
675
Willy Tarreau19d14ef2012-10-29 16:51:55 +0100676 chunk_printf(&trash, "Detected %d consecutive errors, last one was: %s",
677 s->consecutive_errors, get_analyze_status(status));
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100678
679 switch (s->onerror) {
680 case HANA_ONERR_FASTINTER:
681 /* force fastinter - nothing to do here as all modes force it */
682 break;
683
684 case HANA_ONERR_SUDDTH:
685 /* simulate a pre-fatal failed health check */
Simon Horman58c32972013-11-25 10:46:38 +0900686 if (s->check.health > s->check.rise)
687 s->check.health = s->check.rise + 1;
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100688
689 /* no break - fall through */
690
691 case HANA_ONERR_FAILCHK:
692 /* simulate a failed health check */
Simon Horman4a741432013-02-23 15:35:38 +0900693 set_server_check_status(&s->check, HCHK_STATUS_HANA, trash.str);
Simon Hormand8583062013-11-25 10:46:33 +0900694 check_failed(&s->check);
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100695
696 break;
697
698 case HANA_ONERR_MARKDWN:
699 /* mark server down */
Simon Horman58c32972013-11-25 10:46:38 +0900700 s->check.health = s->check.rise;
Simon Horman4a741432013-02-23 15:35:38 +0900701 set_server_check_status(&s->check, HCHK_STATUS_HANA, trash.str);
702 set_server_down(&s->check);
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100703
704 break;
705
706 default:
707 /* write a warning? */
708 break;
709 }
710
711 s->consecutive_errors = 0;
712 s->counters.failed_hana++;
713
Simon Horman66183002013-02-23 10:16:43 +0900714 if (s->check.fastinter) {
715 expire = tick_add(now_ms, MS_TO_TICKS(s->check.fastinter));
Sergiy Prykhodko1d57e502013-09-21 12:05:00 +0300716 if (s->check.task->expire > expire) {
Willy Tarreau5b3a2022012-09-28 15:01:02 +0200717 s->check.task->expire = expire;
Sergiy Prykhodko1d57e502013-09-21 12:05:00 +0300718 /* requeue check task with new expire */
719 task_queue(s->check.task);
720 }
Krzysztof Piotr Oledzki97f07b82009-12-15 22:31:24 +0100721 }
722}
723
Willy Tarreauef781042010-01-27 11:53:01 +0100724static int httpchk_build_status_header(struct server *s, char *buffer)
725{
726 int sv_state;
727 int ratio;
728 int hlen = 0;
729 const char *srv_hlt_st[7] = { "DOWN", "DOWN %d/%d",
730 "UP %d/%d", "UP",
731 "NOLB %d/%d", "NOLB",
732 "no check" };
733
734 memcpy(buffer + hlen, "X-Haproxy-Server-State: ", 24);
735 hlen += 24;
736
737 if (!(s->state & SRV_CHECKED))
738 sv_state = 6; /* should obviously never happen */
739 else if (s->state & SRV_RUNNING) {
Simon Horman58c32972013-11-25 10:46:38 +0900740 if (s->check.health == s->check.rise + s->check.fall - 1)
Willy Tarreauef781042010-01-27 11:53:01 +0100741 sv_state = 3; /* UP */
742 else
743 sv_state = 2; /* going down */
744
745 if (s->state & SRV_GOINGDOWN)
746 sv_state += 2;
747 } else {
Simon Horman125d0992013-02-24 17:23:38 +0900748 if (s->check.health)
Willy Tarreauef781042010-01-27 11:53:01 +0100749 sv_state = 1; /* going up */
750 else
751 sv_state = 0; /* DOWN */
752 }
753
754 hlen += sprintf(buffer + hlen,
755 srv_hlt_st[sv_state],
Simon Horman58c32972013-11-25 10:46:38 +0900756 (s->state & SRV_RUNNING) ? (s->check.health - s->check.rise + 1) : (s->check.health),
757 (s->state & SRV_RUNNING) ? (s->check.fall) : (s->check.rise));
Willy Tarreauef781042010-01-27 11:53:01 +0100758
759 hlen += sprintf(buffer + hlen, "; name=%s/%s; node=%s; weight=%d/%d; scur=%d/%d; qcur=%d",
760 s->proxy->id, s->id,
761 global.node,
762 (s->eweight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv,
763 (s->proxy->lbprm.tot_weight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv,
764 s->cur_sess, s->proxy->beconn - s->proxy->nbpend,
765 s->nbpend);
766
767 if ((s->state & SRV_WARMINGUP) &&
768 now.tv_sec < s->last_change + s->slowstart &&
769 now.tv_sec >= s->last_change) {
770 ratio = MAX(1, 100 * (now.tv_sec - s->last_change) / s->slowstart);
771 hlen += sprintf(buffer + hlen, "; throttle=%d%%", ratio);
772 }
773
774 buffer[hlen++] = '\r';
775 buffer[hlen++] = '\n';
776
777 return hlen;
778}
779
Willy Tarreaubaaee002006-06-26 02:48:02 +0200780/*
781 * This function is used only for server health-checks. It handles
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200782 * the connection acknowledgement. If the proxy requires L7 health-checks,
783 * it sends the request. In other cases, it calls set_server_check_status()
Simon Horman4a741432013-02-23 15:35:38 +0900784 * to set check->status, check->duration and check->result.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200785 */
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200786static void event_srv_chk_w(struct connection *conn)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200787{
Simon Horman4a741432013-02-23 15:35:38 +0900788 struct check *check = conn->owner;
789 struct server *s = check->server;
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200790 int fd = conn->t.sock.fd;
Simon Horman4a741432013-02-23 15:35:38 +0900791 struct task *t = check->task;
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200792
Willy Tarreau5a78f362012-11-23 12:47:05 +0100793 if (conn->flags & (CO_FL_SOCK_WR_SH | CO_FL_DATA_WR_SH))
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200794 conn->flags |= CO_FL_ERROR;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200795
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200796 if (unlikely(conn->flags & CO_FL_ERROR)) {
Krzysztof Piotr Oledzki6492db52010-01-02 22:03:01 +0100797 int skerr, err = errno;
798 socklen_t lskerr = sizeof(skerr);
799
800 if (!getsockopt(fd, SOL_SOCKET, SO_ERROR, &skerr, &lskerr) && skerr)
801 err = skerr;
802
Simon Horman4a741432013-02-23 15:35:38 +0900803 set_server_check_status(check, HCHK_STATUS_L4CON, strerror(err));
Willy Tarreau6996e152007-04-30 14:37:43 +0200804 goto out_error;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200805 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200806
Willy Tarreau5a78f362012-11-23 12:47:05 +0100807 if (conn->flags & (CO_FL_HANDSHAKE | CO_FL_WAIT_WR))
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200808 return;
Willy Tarreau83749182007-04-15 20:56:27 +0200809
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200810 /* here, we know that the connection is established */
Simon Horman4a741432013-02-23 15:35:38 +0900811 if (!(check->result & SRV_CHK_FAILED)) {
Willy Tarreaubaaee002006-06-26 02:48:02 +0200812 /* we don't want to mark 'UP' a server on which we detected an error earlier */
Simon Horman4a741432013-02-23 15:35:38 +0900813 if (check->bo->o) {
814 conn->xprt->snd_buf(conn, check->bo, MSG_DONTWAIT | MSG_NOSIGNAL);
Willy Tarreauf1503172012-09-28 19:39:36 +0200815 if (conn->flags & CO_FL_ERROR) {
Simon Horman4a741432013-02-23 15:35:38 +0900816 set_server_check_status(check, HCHK_STATUS_L4CON, strerror(errno));
Willy Tarreauf1503172012-09-28 19:39:36 +0200817 goto out_wakeup;
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200818 }
Simon Horman4a741432013-02-23 15:35:38 +0900819 if (check->bo->o) {
Willy Tarreau5a78f362012-11-23 12:47:05 +0100820 goto out_incomplete;
821 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200822 }
Willy Tarreau6996e152007-04-30 14:37:43 +0200823
Willy Tarreau5a78f362012-11-23 12:47:05 +0100824 /* full request sent, we allow up to <timeout.check> if nonzero for a response */
825 if (s->proxy->timeout.check) {
826 t->expire = tick_add_ifset(now_ms, s->proxy->timeout.check);
827 task_queue(t);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200828 }
Willy Tarreau5a78f362012-11-23 12:47:05 +0100829 goto out_nowake;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200830 }
Willy Tarreau83749182007-04-15 20:56:27 +0200831 out_wakeup:
Willy Tarreaufdccded2008-08-29 18:19:04 +0200832 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreau83749182007-04-15 20:56:27 +0200833 out_nowake:
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200834 __conn_data_stop_send(conn); /* nothing more to write */
Willy Tarreau5a78f362012-11-23 12:47:05 +0100835 out_incomplete:
Willy Tarreau3267d362012-08-17 23:53:56 +0200836 return;
Willy Tarreau6996e152007-04-30 14:37:43 +0200837 out_error:
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200838 conn->flags |= CO_FL_ERROR;
Willy Tarreau6996e152007-04-30 14:37:43 +0200839 goto out_wakeup;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200840}
841
Willy Tarreaubaaee002006-06-26 02:48:02 +0200842/*
Willy Tarreauf3c69202006-07-09 16:42:34 +0200843 * This function is used only for server health-checks. It handles the server's
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +0200844 * reply to an HTTP request, SSL HELLO or MySQL client Auth. It calls
Simon Horman4a741432013-02-23 15:35:38 +0900845 * set_server_check_status() to update check->status, check->duration
846 * and check->result.
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200847
848 * The set_server_check_status function is called with HCHK_STATUS_L7OKD if
849 * an HTTP server replies HTTP 2xx or 3xx (valid responses), if an SMTP server
850 * returns 2xx, HCHK_STATUS_L6OK if an SSL server returns at least 5 bytes in
851 * response to an SSL HELLO (the principle is that this is enough to
852 * distinguish between an SSL server and a pure TCP relay). All other cases will
853 * call it with a proper error status like HCHK_STATUS_L7STS, HCHK_STATUS_L6RSP,
854 * etc.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200855 */
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200856static void event_srv_chk_r(struct connection *conn)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200857{
Simon Horman4a741432013-02-23 15:35:38 +0900858 struct check *check = conn->owner;
859 struct server *s = check->server;
860 struct task *t = check->task;
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200861 char *desc;
Willy Tarreau03938182010-03-17 21:52:07 +0100862 int done;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +0200863 unsigned short msglen;
Willy Tarreau83749182007-04-15 20:56:27 +0200864
Simon Horman4a741432013-02-23 15:35:38 +0900865 if (unlikely((check->result & SRV_CHK_FAILED) || (conn->flags & CO_FL_ERROR))) {
Willy Tarreau83749182007-04-15 20:56:27 +0200866 /* in case of TCP only, this tells us if the connection failed */
Simon Horman4a741432013-02-23 15:35:38 +0900867 if (!(check->result & SRV_CHK_FAILED))
868 set_server_check_status(check, HCHK_STATUS_SOCKERR, NULL);
Willy Tarreau83749182007-04-15 20:56:27 +0200869 goto out_wakeup;
870 }
871
Willy Tarreau5a78f362012-11-23 12:47:05 +0100872 if (conn->flags & (CO_FL_HANDSHAKE | CO_FL_WAIT_RD))
Willy Tarreaufb56aab2012-09-28 14:40:02 +0200873 return;
874
Willy Tarreau83749182007-04-15 20:56:27 +0200875 /* Warning! Linux returns EAGAIN on SO_ERROR if data are still available
876 * but the connection was closed on the remote end. Fortunately, recv still
877 * works correctly and we don't need to do the getsockopt() on linux.
878 */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000879
880 /* Set buffer to point to the end of the data already read, and check
881 * that there is free space remaining. If the buffer is full, proceed
882 * with running the checks without attempting another socket read.
883 */
Nick Chalk57b1bf72010-03-16 15:50:46 +0000884
Willy Tarreau03938182010-03-17 21:52:07 +0100885 done = 0;
Nick Chalk57b1bf72010-03-16 15:50:46 +0000886
Simon Horman4a741432013-02-23 15:35:38 +0900887 conn->xprt->rcv_buf(conn, check->bi, check->bi->size);
Willy Tarreauf1503172012-09-28 19:39:36 +0200888 if (conn->flags & (CO_FL_ERROR | CO_FL_SOCK_RD_SH | CO_FL_DATA_RD_SH)) {
Willy Tarreau03938182010-03-17 21:52:07 +0100889 done = 1;
Simon Horman4a741432013-02-23 15:35:38 +0900890 if ((conn->flags & CO_FL_ERROR) && !check->bi->i) {
Willy Tarreauf1503172012-09-28 19:39:36 +0200891 /* Report network errors only if we got no other data. Otherwise
892 * we'll let the upper layers decide whether the response is OK
893 * or not. It is very common that an RST sent by the server is
894 * reported as an error just after the last data chunk.
895 */
Simon Horman4a741432013-02-23 15:35:38 +0900896 if (!(check->result & SRV_CHK_FAILED))
897 set_server_check_status(check, HCHK_STATUS_SOCKERR, NULL);
Willy Tarreauc1a07962010-03-16 20:55:43 +0100898 goto out_wakeup;
899 }
Willy Tarreaubaaee002006-06-26 02:48:02 +0200900 }
901
Willy Tarreau03938182010-03-17 21:52:07 +0100902 /* Intermediate or complete response received.
Simon Horman4a741432013-02-23 15:35:38 +0900903 * Terminate string in check->bi->data buffer.
Willy Tarreau03938182010-03-17 21:52:07 +0100904 */
Simon Horman4a741432013-02-23 15:35:38 +0900905 if (check->bi->i < check->bi->size)
906 check->bi->data[check->bi->i] = '\0';
Willy Tarreau03938182010-03-17 21:52:07 +0100907 else {
Simon Horman4a741432013-02-23 15:35:38 +0900908 check->bi->data[check->bi->i - 1] = '\0';
Willy Tarreau03938182010-03-17 21:52:07 +0100909 done = 1; /* buffer full, don't wait for more data */
910 }
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200911
Nick Chalk57b1bf72010-03-16 15:50:46 +0000912 /* Run the checks... */
Simon Horman4a741432013-02-23 15:35:38 +0900913 switch (check->type) {
Willy Tarreau1620ec32011-08-06 17:05:02 +0200914 case PR_O2_HTTP_CHK:
Simon Horman4a741432013-02-23 15:35:38 +0900915 if (!done && check->bi->i < strlen("HTTP/1.0 000\r"))
Willy Tarreau03938182010-03-17 21:52:07 +0100916 goto wait_more_data;
917
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100918 /* Check if the server speaks HTTP 1.X */
Simon Horman4a741432013-02-23 15:35:38 +0900919 if ((check->bi->i < strlen("HTTP/1.0 000\r")) ||
920 (memcmp(check->bi->data, "HTTP/1.", 7) != 0 ||
921 (*(check->bi->data + 12) != ' ' && *(check->bi->data + 12) != '\r')) ||
922 !isdigit((unsigned char) *(check->bi->data + 9)) || !isdigit((unsigned char) *(check->bi->data + 10)) ||
923 !isdigit((unsigned char) *(check->bi->data + 11))) {
924 cut_crlf(check->bi->data);
925 set_server_check_status(check, HCHK_STATUS_L7RSP, check->bi->data);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200926
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100927 goto out_wakeup;
928 }
929
Simon Horman4a741432013-02-23 15:35:38 +0900930 check->code = str2uic(check->bi->data + 9);
931 desc = ltrim(check->bi->data + 12, ' ');
Nick Chalk57b1bf72010-03-16 15:50:46 +0000932
Willy Tarreaubd741542010-03-16 18:46:54 +0100933 if ((s->proxy->options & PR_O_DISABLE404) &&
Simon Horman4a741432013-02-23 15:35:38 +0900934 (s->state & SRV_RUNNING) && (check->code == 404)) {
Nick Chalk57b1bf72010-03-16 15:50:46 +0000935 /* 404 may be accepted as "stopping" only if the server was up */
936 cut_crlf(desc);
Simon Horman4a741432013-02-23 15:35:38 +0900937 set_server_check_status(check, HCHK_STATUS_L7OKCD, desc);
Nick Chalk57b1bf72010-03-16 15:50:46 +0000938 }
Willy Tarreaubd741542010-03-16 18:46:54 +0100939 else if (s->proxy->options2 & PR_O2_EXP_TYPE) {
940 /* Run content verification check... We know we have at least 13 chars */
941 if (!httpchk_expect(s, done))
942 goto wait_more_data;
943 }
944 /* check the reply : HTTP/1.X 2xx and 3xx are OK */
Simon Horman4a741432013-02-23 15:35:38 +0900945 else if (*(check->bi->data + 9) == '2' || *(check->bi->data + 9) == '3') {
Willy Tarreaubd741542010-03-16 18:46:54 +0100946 cut_crlf(desc);
Simon Horman4a741432013-02-23 15:35:38 +0900947 set_server_check_status(check, HCHK_STATUS_L7OKD, desc);
Willy Tarreaubd741542010-03-16 18:46:54 +0100948 }
Nick Chalk57b1bf72010-03-16 15:50:46 +0000949 else {
950 cut_crlf(desc);
Simon Horman4a741432013-02-23 15:35:38 +0900951 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Nick Chalk57b1bf72010-03-16 15:50:46 +0000952 }
Willy Tarreau1620ec32011-08-06 17:05:02 +0200953 break;
954
955 case PR_O2_SSL3_CHK:
Simon Horman4a741432013-02-23 15:35:38 +0900956 if (!done && check->bi->i < 5)
Willy Tarreau03938182010-03-17 21:52:07 +0100957 goto wait_more_data;
958
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100959 /* Check for SSLv3 alert or handshake */
Simon Horman4a741432013-02-23 15:35:38 +0900960 if ((check->bi->i >= 5) && (*check->bi->data == 0x15 || *check->bi->data == 0x16))
961 set_server_check_status(check, HCHK_STATUS_L6OK, NULL);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200962 else
Simon Horman4a741432013-02-23 15:35:38 +0900963 set_server_check_status(check, HCHK_STATUS_L6RSP, NULL);
Willy Tarreau1620ec32011-08-06 17:05:02 +0200964 break;
965
966 case PR_O2_SMTP_CHK:
Simon Horman4a741432013-02-23 15:35:38 +0900967 if (!done && check->bi->i < strlen("000\r"))
Willy Tarreau03938182010-03-17 21:52:07 +0100968 goto wait_more_data;
969
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200970 /* Check if the server speaks SMTP */
Simon Horman4a741432013-02-23 15:35:38 +0900971 if ((check->bi->i < strlen("000\r")) ||
972 (*(check->bi->data + 3) != ' ' && *(check->bi->data + 3) != '\r') ||
973 !isdigit((unsigned char) *check->bi->data) || !isdigit((unsigned char) *(check->bi->data + 1)) ||
974 !isdigit((unsigned char) *(check->bi->data + 2))) {
975 cut_crlf(check->bi->data);
976 set_server_check_status(check, HCHK_STATUS_L7RSP, check->bi->data);
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200977
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200978 goto out_wakeup;
979 }
980
Simon Horman4a741432013-02-23 15:35:38 +0900981 check->code = str2uic(check->bi->data);
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +0200982
Simon Horman4a741432013-02-23 15:35:38 +0900983 desc = ltrim(check->bi->data + 3, ' ');
Krzysztof Piotr Oledzkif7089f52009-10-10 21:06:49 +0200984 cut_crlf(desc);
985
Willy Tarreauc7dd71a2007-11-30 08:33:21 +0100986 /* Check for SMTP code 2xx (should be 250) */
Simon Horman4a741432013-02-23 15:35:38 +0900987 if (*check->bi->data == '2')
988 set_server_check_status(check, HCHK_STATUS_L7OKD, desc);
Krzysztof Piotr Oledzki213014e2009-09-27 15:50:02 +0200989 else
Simon Horman4a741432013-02-23 15:35:38 +0900990 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Willy Tarreau1620ec32011-08-06 17:05:02 +0200991 break;
992
Simon Hormana2b9dad2013-02-12 10:45:54 +0900993 case PR_O2_LB_AGENT_CHK: {
994 short status = HCHK_STATUS_L7RSP;
995 const char *desc = "Unknown feedback string";
996 const char *down_cmd = NULL;
Simon Horman671b6f02013-11-25 10:46:39 +0900997 int disabled;
Simon Hormana2b9dad2013-02-12 10:45:54 +0900998
999 if (!done)
1000 goto wait_more_data;
1001
Simon Horman4a741432013-02-23 15:35:38 +09001002 cut_crlf(check->bi->data);
Simon Hormana2b9dad2013-02-12 10:45:54 +09001003
Simon Horman671b6f02013-11-25 10:46:39 +09001004 /*
1005 * The agent may have been disabled after a check was
1006 * initialised. If so, ignore weight changes and drain
1007 * settings from the agent. Note that the setting is
1008 * always present in the state of the agent the server,
1009 * regardless of if the agent is being run as a primary or
1010 * secondary check. That is, regardless of if the check
1011 * parameter of this function is the agent or check field
1012 * of the server.
1013 */
1014 disabled = check->server->agent.state & CHK_STATE_DISABLED;
1015
Simon Horman4a741432013-02-23 15:35:38 +09001016 if (strchr(check->bi->data, '%')) {
Simon Horman671b6f02013-11-25 10:46:39 +09001017 if (disabled)
1018 break;
Simon Horman4a741432013-02-23 15:35:38 +09001019 desc = server_parse_weight_change_request(s, check->bi->data);
Simon Hormana2b9dad2013-02-12 10:45:54 +09001020 if (!desc) {
1021 status = HCHK_STATUS_L7OKD;
Simon Horman4a741432013-02-23 15:35:38 +09001022 desc = check->bi->data;
Simon Hormana2b9dad2013-02-12 10:45:54 +09001023 }
Simon Horman4a741432013-02-23 15:35:38 +09001024 } else if (!strcasecmp(check->bi->data, "drain")) {
Simon Horman671b6f02013-11-25 10:46:39 +09001025 if (disabled)
1026 break;
Simon Hormana2b9dad2013-02-12 10:45:54 +09001027 desc = server_parse_weight_change_request(s, "0%");
1028 if (!desc) {
1029 desc = "drain";
1030 status = HCHK_STATUS_L7OKD;
1031 }
Simon Horman4a741432013-02-23 15:35:38 +09001032 } else if (!strncasecmp(check->bi->data, "down", strlen("down"))) {
Simon Hormana2b9dad2013-02-12 10:45:54 +09001033 down_cmd = "down";
Simon Horman4a741432013-02-23 15:35:38 +09001034 } else if (!strncasecmp(check->bi->data, "stopped", strlen("stopped"))) {
Simon Hormana2b9dad2013-02-12 10:45:54 +09001035 down_cmd = "stopped";
Simon Horman4a741432013-02-23 15:35:38 +09001036 } else if (!strncasecmp(check->bi->data, "fail", strlen("fail"))) {
Simon Hormana2b9dad2013-02-12 10:45:54 +09001037 down_cmd = "fail";
1038 }
1039
1040 if (down_cmd) {
Simon Horman4a741432013-02-23 15:35:38 +09001041 const char *end = check->bi->data + strlen(down_cmd);
Simon Hormana2b9dad2013-02-12 10:45:54 +09001042 /*
1043 * The command keyword must terminated the string or
1044 * be followed by a blank.
1045 */
Willy Tarreau8b4c3762013-02-13 12:47:12 +01001046 if (end[0] == '\0' || end[0] == ' ' || end[0] == '\t') {
Simon Hormana2b9dad2013-02-12 10:45:54 +09001047 status = HCHK_STATUS_L7STS;
Simon Horman80fefae2013-11-25 10:46:34 +09001048 desc = check->bi->data;
Simon Hormana2b9dad2013-02-12 10:45:54 +09001049 }
1050 }
1051
Simon Horman4a741432013-02-23 15:35:38 +09001052 set_server_check_status(check, status, desc);
Simon Horman8c3d0be2013-11-25 10:46:40 +09001053 set_server_drain_state(check->server);
Simon Hormana2b9dad2013-02-12 10:45:54 +09001054 break;
1055 }
1056
Willy Tarreau1620ec32011-08-06 17:05:02 +02001057 case PR_O2_PGSQL_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001058 if (!done && check->bi->i < 9)
Rauf Kuliyev38b41562011-01-04 15:14:13 +01001059 goto wait_more_data;
1060
Simon Horman4a741432013-02-23 15:35:38 +09001061 if (check->bi->data[0] == 'R') {
1062 set_server_check_status(check, HCHK_STATUS_L7OKD, "PostgreSQL server is ok");
Rauf Kuliyev38b41562011-01-04 15:14:13 +01001063 }
1064 else {
Simon Horman4a741432013-02-23 15:35:38 +09001065 if ((check->bi->data[0] == 'E') && (check->bi->data[5]!=0) && (check->bi->data[6]!=0))
1066 desc = &check->bi->data[6];
Rauf Kuliyev38b41562011-01-04 15:14:13 +01001067 else
1068 desc = "PostgreSQL unknown error";
1069
Simon Horman4a741432013-02-23 15:35:38 +09001070 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Rauf Kuliyev38b41562011-01-04 15:14:13 +01001071 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001072 break;
1073
1074 case PR_O2_REDIS_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001075 if (!done && check->bi->i < 7)
Hervé COMMOWICKec032d62011-08-05 16:23:48 +02001076 goto wait_more_data;
1077
Simon Horman4a741432013-02-23 15:35:38 +09001078 if (strcmp(check->bi->data, "+PONG\r\n") == 0) {
1079 set_server_check_status(check, HCHK_STATUS_L7OKD, "Redis server is ok");
Hervé COMMOWICKec032d62011-08-05 16:23:48 +02001080 }
1081 else {
Simon Horman4a741432013-02-23 15:35:38 +09001082 set_server_check_status(check, HCHK_STATUS_L7STS, check->bi->data);
Hervé COMMOWICKec032d62011-08-05 16:23:48 +02001083 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001084 break;
1085
1086 case PR_O2_MYSQL_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001087 if (!done && check->bi->i < 5)
Willy Tarreau03938182010-03-17 21:52:07 +01001088 goto wait_more_data;
1089
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001090 if (s->proxy->check_len == 0) { // old mode
Simon Horman4a741432013-02-23 15:35:38 +09001091 if (*(check->bi->data + 4) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001092 /* We set the MySQL Version in description for information purpose
1093 * FIXME : it can be cool to use MySQL Version for other purpose,
1094 * like mark as down old MySQL server.
1095 */
Simon Horman4a741432013-02-23 15:35:38 +09001096 if (check->bi->i > 51) {
1097 desc = ltrim(check->bi->data + 5, ' ');
1098 set_server_check_status(check, HCHK_STATUS_L7OKD, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001099 }
1100 else {
1101 if (!done)
1102 goto wait_more_data;
1103 /* it seems we have a OK packet but without a valid length,
1104 * it must be a protocol error
1105 */
Simon Horman4a741432013-02-23 15:35:38 +09001106 set_server_check_status(check, HCHK_STATUS_L7RSP, check->bi->data);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001107 }
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001108 }
1109 else {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001110 /* An error message is attached in the Error packet */
Simon Horman4a741432013-02-23 15:35:38 +09001111 desc = ltrim(check->bi->data + 7, ' ');
1112 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001113 }
1114 } else {
Simon Horman4a741432013-02-23 15:35:38 +09001115 unsigned int first_packet_len = ((unsigned int) *check->bi->data) +
1116 (((unsigned int) *(check->bi->data + 1)) << 8) +
1117 (((unsigned int) *(check->bi->data + 2)) << 16);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001118
Simon Horman4a741432013-02-23 15:35:38 +09001119 if (check->bi->i == first_packet_len + 4) {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001120 /* MySQL Error packet always begin with field_count = 0xff */
Simon Horman4a741432013-02-23 15:35:38 +09001121 if (*(check->bi->data + 4) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001122 /* We have only one MySQL packet and it is a Handshake Initialization packet
1123 * but we need to have a second packet to know if it is alright
1124 */
Simon Horman4a741432013-02-23 15:35:38 +09001125 if (!done && check->bi->i < first_packet_len + 5)
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001126 goto wait_more_data;
1127 }
1128 else {
1129 /* We have only one packet and it is an Error packet,
1130 * an error message is attached, so we can display it
1131 */
Simon Horman4a741432013-02-23 15:35:38 +09001132 desc = &check->bi->data[7];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001133 //Warning("onlyoneERR: %s\n", desc);
Simon Horman4a741432013-02-23 15:35:38 +09001134 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001135 }
Simon Horman4a741432013-02-23 15:35:38 +09001136 } else if (check->bi->i > first_packet_len + 4) {
1137 unsigned int second_packet_len = ((unsigned int) *(check->bi->data + first_packet_len + 4)) +
1138 (((unsigned int) *(check->bi->data + first_packet_len + 5)) << 8) +
1139 (((unsigned int) *(check->bi->data + first_packet_len + 6)) << 16);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001140
Simon Horman4a741432013-02-23 15:35:38 +09001141 if (check->bi->i == first_packet_len + 4 + second_packet_len + 4 ) {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001142 /* We have 2 packets and that's good */
1143 /* Check if the second packet is a MySQL Error packet or not */
Simon Horman4a741432013-02-23 15:35:38 +09001144 if (*(check->bi->data + first_packet_len + 8) != '\xff') {
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001145 /* No error packet */
1146 /* We set the MySQL Version in description for information purpose */
Simon Horman4a741432013-02-23 15:35:38 +09001147 desc = &check->bi->data[5];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001148 //Warning("2packetOK: %s\n", desc);
Simon Horman4a741432013-02-23 15:35:38 +09001149 set_server_check_status(check, HCHK_STATUS_L7OKD, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001150 }
1151 else {
1152 /* An error message is attached in the Error packet
1153 * so we can display it ! :)
1154 */
Simon Horman4a741432013-02-23 15:35:38 +09001155 desc = &check->bi->data[first_packet_len+11];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001156 //Warning("2packetERR: %s\n", desc);
Simon Horman4a741432013-02-23 15:35:38 +09001157 set_server_check_status(check, HCHK_STATUS_L7STS, desc);
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001158 }
1159 }
1160 }
1161 else {
Willy Tarreau03938182010-03-17 21:52:07 +01001162 if (!done)
1163 goto wait_more_data;
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001164 /* it seems we have a Handshake Initialization packet but without a valid length,
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001165 * it must be a protocol error
1166 */
Simon Horman4a741432013-02-23 15:35:38 +09001167 desc = &check->bi->data[5];
Hervé COMMOWICK8776f1b2010-10-18 15:58:36 +02001168 //Warning("protoerr: %s\n", desc);
Simon Horman4a741432013-02-23 15:35:38 +09001169 set_server_check_status(check, HCHK_STATUS_L7RSP, desc);
Hervé COMMOWICK698ae002010-01-12 09:25:13 +01001170 }
1171 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001172 break;
1173
1174 case PR_O2_LDAP_CHK:
Simon Horman4a741432013-02-23 15:35:38 +09001175 if (!done && check->bi->i < 14)
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001176 goto wait_more_data;
1177
1178 /* Check if the server speaks LDAP (ASN.1/BER)
1179 * http://en.wikipedia.org/wiki/Basic_Encoding_Rules
1180 * http://tools.ietf.org/html/rfc4511
1181 */
1182
1183 /* http://tools.ietf.org/html/rfc4511#section-4.1.1
1184 * LDAPMessage: 0x30: SEQUENCE
1185 */
Simon Horman4a741432013-02-23 15:35:38 +09001186 if ((check->bi->i < 14) || (*(check->bi->data) != '\x30')) {
1187 set_server_check_status(check, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001188 }
1189 else {
1190 /* size of LDAPMessage */
Simon Horman4a741432013-02-23 15:35:38 +09001191 msglen = (*(check->bi->data + 1) & 0x80) ? (*(check->bi->data + 1) & 0x7f) : 0;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001192
1193 /* http://tools.ietf.org/html/rfc4511#section-4.2.2
1194 * messageID: 0x02 0x01 0x01: INTEGER 1
1195 * protocolOp: 0x61: bindResponse
1196 */
1197 if ((msglen > 2) ||
Simon Horman4a741432013-02-23 15:35:38 +09001198 (memcmp(check->bi->data + 2 + msglen, "\x02\x01\x01\x61", 4) != 0)) {
1199 set_server_check_status(check, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001200
1201 goto out_wakeup;
1202 }
1203
1204 /* size of bindResponse */
Simon Horman4a741432013-02-23 15:35:38 +09001205 msglen += (*(check->bi->data + msglen + 6) & 0x80) ? (*(check->bi->data + msglen + 6) & 0x7f) : 0;
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001206
1207 /* http://tools.ietf.org/html/rfc4511#section-4.1.9
1208 * ldapResult: 0x0a 0x01: ENUMERATION
1209 */
1210 if ((msglen > 4) ||
Simon Horman4a741432013-02-23 15:35:38 +09001211 (memcmp(check->bi->data + 7 + msglen, "\x0a\x01", 2) != 0)) {
1212 set_server_check_status(check, HCHK_STATUS_L7RSP, "Not LDAPv3 protocol");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001213
1214 goto out_wakeup;
1215 }
1216
1217 /* http://tools.ietf.org/html/rfc4511#section-4.1.9
1218 * resultCode
1219 */
Simon Horman4a741432013-02-23 15:35:38 +09001220 check->code = *(check->bi->data + msglen + 9);
1221 if (check->code) {
1222 set_server_check_status(check, HCHK_STATUS_L7STS, "See RFC: http://tools.ietf.org/html/rfc4511#section-4.1.9");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001223 } else {
Simon Horman4a741432013-02-23 15:35:38 +09001224 set_server_check_status(check, HCHK_STATUS_L7OKD, "Success");
Gabor Lekenyb4c81e42010-09-29 18:17:05 +02001225 }
1226 }
Willy Tarreau1620ec32011-08-06 17:05:02 +02001227 break;
1228
1229 default:
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001230 /* other checks are valid if the connection succeeded anyway */
Simon Horman4a741432013-02-23 15:35:38 +09001231 set_server_check_status(check, HCHK_STATUS_L4OK, NULL);
Willy Tarreau1620ec32011-08-06 17:05:02 +02001232 break;
1233 } /* switch */
Willy Tarreau83749182007-04-15 20:56:27 +02001234
Willy Tarreauc7dd71a2007-11-30 08:33:21 +01001235 out_wakeup:
Simon Horman4a741432013-02-23 15:35:38 +09001236 if (check->result & SRV_CHK_FAILED)
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001237 conn->flags |= CO_FL_ERROR;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001238
Nick Chalk57b1bf72010-03-16 15:50:46 +00001239 /* Reset the check buffer... */
Simon Horman4a741432013-02-23 15:35:38 +09001240 *check->bi->data = '\0';
1241 check->bi->i = 0;
Nick Chalk57b1bf72010-03-16 15:50:46 +00001242
Willy Tarreaufd29cc52012-11-23 09:18:20 +01001243 /* Close the connection... We absolutely want to perform a hard close
1244 * and reset the connection if some data are pending, otherwise we end
1245 * up with many TIME_WAITs and eat all the source port range quickly.
1246 * To avoid sending RSTs all the time, we first try to drain pending
1247 * data.
1248 */
Willy Tarreauf1503172012-09-28 19:39:36 +02001249 if (conn->xprt && conn->xprt->shutw)
1250 conn->xprt->shutw(conn, 0);
Willy Tarreau2b57cb82013-06-10 19:56:38 +02001251
1252 if (conn->ctrl && !(conn->flags & CO_FL_SOCK_RD_SH)) {
1253 if (conn->flags & CO_FL_WAIT_RD || !conn->ctrl->drain || !conn->ctrl->drain(conn->t.sock.fd))
1254 setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
1255 (struct linger *) &nolinger, sizeof(struct linger));
Willy Tarreaucfd97c62012-11-23 17:35:59 +01001256 }
Willy Tarreaua522f802012-11-23 08:56:35 +01001257 __conn_data_stop_both(conn);
Willy Tarreaufdccded2008-08-29 18:19:04 +02001258 task_wakeup(t, TASK_WOKEN_IO);
Willy Tarreau3267d362012-08-17 23:53:56 +02001259 return;
Willy Tarreau03938182010-03-17 21:52:07 +01001260
1261 wait_more_data:
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001262 __conn_data_poll_recv(conn);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001263}
1264
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001265/*
1266 * This function is used only for server health-checks. It handles connection
1267 * status updates including errors. If necessary, it wakes the check task up.
1268 * It always returns 0.
1269 */
1270static int wake_srv_chk(struct connection *conn)
Willy Tarreau20bea422012-07-06 12:00:49 +02001271{
Simon Horman4a741432013-02-23 15:35:38 +09001272 struct check *check = conn->owner;
Willy Tarreau20bea422012-07-06 12:00:49 +02001273
Willy Tarreau6c560da2012-11-24 11:14:45 +01001274 if (unlikely(conn->flags & CO_FL_ERROR)) {
1275 /* Note that we might as well have been woken up by a handshake handler */
Simon Horman4a741432013-02-23 15:35:38 +09001276 if (check->result == SRV_CHK_UNKNOWN)
1277 check->result |= SRV_CHK_FAILED;
Willy Tarreau6c560da2012-11-24 11:14:45 +01001278 __conn_data_stop_both(conn);
Simon Horman4a741432013-02-23 15:35:38 +09001279 task_wakeup(check->task, TASK_WOKEN_IO);
Willy Tarreau6c560da2012-11-24 11:14:45 +01001280 }
Willy Tarreau20bea422012-07-06 12:00:49 +02001281
Simon Horman4a741432013-02-23 15:35:38 +09001282 if (check->result & (SRV_CHK_FAILED|SRV_CHK_PASSED))
Willy Tarreau2b199c92012-11-23 17:32:21 +01001283 conn_full_close(conn);
Willy Tarreau3267d362012-08-17 23:53:56 +02001284 return 0;
Willy Tarreau20bea422012-07-06 12:00:49 +02001285}
1286
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001287struct data_cb check_conn_cb = {
1288 .recv = event_srv_chk_r,
1289 .send = event_srv_chk_w,
1290 .wake = wake_srv_chk,
1291};
1292
Willy Tarreaubaaee002006-06-26 02:48:02 +02001293/*
Willy Tarreau2e993902011-10-31 11:53:20 +01001294 * updates the server's weight during a warmup stage. Once the final weight is
1295 * reached, the task automatically stops. Note that any server status change
1296 * must have updated s->last_change accordingly.
1297 */
1298static struct task *server_warmup(struct task *t)
1299{
1300 struct server *s = t->context;
1301
1302 /* by default, plan on stopping the task */
1303 t->expire = TICK_ETERNITY;
1304 if ((s->state & (SRV_RUNNING|SRV_WARMINGUP|SRV_MAINTAIN)) != (SRV_RUNNING|SRV_WARMINGUP))
1305 return t;
1306
Willy Tarreau004e0452013-11-21 11:22:01 +01001307 server_recalc_eweight(s);
Willy Tarreau2e993902011-10-31 11:53:20 +01001308
1309 /* probably that we can refill this server with a bit more connections */
1310 check_for_pending(s);
1311
1312 /* get back there in 1 second or 1/20th of the slowstart interval,
1313 * whichever is greater, resulting in small 5% steps.
1314 */
1315 if (s->state & SRV_WARMINGUP)
1316 t->expire = tick_add(now_ms, MS_TO_TICKS(MAX(1000, s->slowstart / 20)));
1317 return t;
1318}
1319
1320/*
Willy Tarreaubaaee002006-06-26 02:48:02 +02001321 * manages a server health-check. Returns
1322 * the time the task accepts to wait, or TIME_ETERNITY for infinity.
1323 */
Simon Horman63a4a822012-03-19 07:24:41 +09001324static struct task *process_chk(struct task *t)
Willy Tarreaubaaee002006-06-26 02:48:02 +02001325{
Simon Horman4a741432013-02-23 15:35:38 +09001326 struct check *check = t->context;
1327 struct server *s = check->server;
1328 struct connection *conn = check->conn;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001329 int rv;
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001330 int ret;
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001331 int expired = tick_is_expired(t->expire, now_ms);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001332
Simon Hormancd5d7b62013-02-24 17:23:38 +09001333 if (!(check->state & CHK_STATE_RUNNING)) {
Willy Tarreau5a78f362012-11-23 12:47:05 +01001334 /* no check currently running */
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001335 if (!expired) /* woke up too early */
Willy Tarreau26c25062009-03-08 09:38:41 +01001336 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001337
Simon Horman671b6f02013-11-25 10:46:39 +09001338 /* we don't send any health-checks when the proxy is
1339 * stopped, the server should not be checked or the check
1340 * is disabled.
Willy Tarreaubaaee002006-06-26 02:48:02 +02001341 */
Simon Horman671b6f02013-11-25 10:46:39 +09001342 if (!(s->state & SRV_CHECKED) ||
1343 s->proxy->state == PR_STSTOPPED ||
1344 (s->state & SRV_MAINTAIN) ||
1345 (check->state & CHK_STATE_DISABLED))
Willy Tarreau5a78f362012-11-23 12:47:05 +01001346 goto reschedule;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001347
1348 /* we'll initiate a new check */
Simon Horman4a741432013-02-23 15:35:38 +09001349 set_server_check_status(check, HCHK_STATUS_START, NULL);
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001350
Simon Hormancd5d7b62013-02-24 17:23:38 +09001351 check->state |= CHK_STATE_RUNNING;
Simon Horman4a741432013-02-23 15:35:38 +09001352 check->bi->p = check->bi->data;
1353 check->bi->i = 0;
1354 check->bo->p = check->bo->data;
1355 check->bo->o = 0;
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001356
Simon Hormand60d6912013-11-25 10:46:36 +09001357 /* prepare the check buffer
1358 * This should not be used if check is the secondary agent check
1359 * of a server as s->proxy->check_req will relate to the
1360 * configuration of the primary check */
1361 if (check->type && check != &s->agent) {
Simon Horman4a741432013-02-23 15:35:38 +09001362 bo_putblk(check->bo, s->proxy->check_req, s->proxy->check_len);
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001363
1364 /* we want to check if this host replies to HTTP or SSLv3 requests
1365 * so we'll send the request, and won't wake the checker up now.
1366 */
Simon Horman4a741432013-02-23 15:35:38 +09001367 if ((check->type) == PR_O2_SSL3_CHK) {
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001368 /* SSL requires that we put Unix time in the request */
1369 int gmt_time = htonl(date.tv_sec);
Simon Horman4a741432013-02-23 15:35:38 +09001370 memcpy(check->bo->data + 11, &gmt_time, 4);
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001371 }
Simon Horman4a741432013-02-23 15:35:38 +09001372 else if ((check->type) == PR_O2_HTTP_CHK) {
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001373 if (s->proxy->options2 & PR_O2_CHK_SNDST)
Simon Horman4a741432013-02-23 15:35:38 +09001374 bo_putblk(check->bo, trash.str, httpchk_build_status_header(s, trash.str));
1375 bo_putstr(check->bo, "\r\n");
1376 *check->bo->p = '\0'; /* to make gdb output easier to read */
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001377 }
1378 }
1379
1380 /* prepare a new connection */
Willy Tarreaub63b5962012-11-23 16:22:08 +01001381 conn->flags = CO_FL_NONE;
Willy Tarreau14cba4b2012-11-30 17:33:05 +01001382 conn->err_code = CO_ER_NONE;
Willy Tarreau3fdb3662012-11-12 00:42:33 +01001383 conn->target = &s->obj_type;
Simon Horman4a741432013-02-23 15:35:38 +09001384 conn_prepare(conn, &check_conn_cb, s->check_common.proto, s->check_common.xprt, check);
Willy Tarreauc6f4ce82009-06-10 11:09:37 +02001385
Willy Tarreau5f2877a2012-10-26 19:57:58 +02001386 /* no client address */
1387 clear_addr(&conn->addr.from);
1388
Simon Horman66183002013-02-23 10:16:43 +09001389 if (is_addr(&s->check_common.addr))
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001390 /* we'll connect to the check addr specified on the server */
Simon Horman66183002013-02-23 10:16:43 +09001391 conn->addr.to = s->check_common.addr;
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001392 else
1393 /* we'll connect to the addr on the server */
1394 conn->addr.to = s->addr;
Willy Tarreauc6f4ce82009-06-10 11:09:37 +02001395
Simon Horman4a741432013-02-23 15:35:38 +09001396 set_host_port(&conn->addr.to, check->port);
Willy Tarreaue8c66af2008-01-13 18:40:14 +01001397
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001398 /* It can return one of :
1399 * - SN_ERR_NONE if everything's OK
1400 * - SN_ERR_SRVTO if there are no more servers
1401 * - SN_ERR_SRVCL if the connection was refused by the server
1402 * - SN_ERR_PRXCOND if the connection has been limited by the proxy (maxconn)
1403 * - SN_ERR_RESOURCE if a system resource is lacking (eg: fd limits, ports, ...)
1404 * - SN_ERR_INTERNAL for any other purely internal errors
1405 * Additionnally, in the case of SN_ERR_RESOURCE, an emergency log will be emitted.
Willy Tarreau24db47e2012-11-23 14:16:39 +01001406 * Note that we try to prevent the network stack from sending the ACK during the
Willy Tarreauf0837b22012-11-24 10:24:27 +01001407 * connect() when a pure TCP check is used (without PROXY protocol).
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001408 */
Willy Tarreau8f46cca2013-03-04 20:07:44 +01001409 ret = SN_ERR_INTERNAL;
Simon Horman66183002013-02-23 10:16:43 +09001410 if (s->check_common.proto->connect)
Simon Horman4a741432013-02-23 15:35:38 +09001411 ret = s->check_common.proto->connect(conn, check->type,
1412 s->check.send_proxy ? 1 : (check->type) ? 0 : 2);
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001413 conn->flags |= CO_FL_WAKE_DATA;
Simon Horman4a741432013-02-23 15:35:38 +09001414 if (check->send_proxy)
Willy Tarreau6c16adc2012-10-05 00:04:16 +02001415 conn->flags |= CO_FL_LOCAL_SPROXY;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001416
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001417 switch (ret) {
1418 case SN_ERR_NONE:
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001419 /* we allow up to min(inter, timeout.connect) for a connection
1420 * to establish but only when timeout.check is set
1421 * as it may be to short for a full check otherwise
1422 */
Simon Horman4a741432013-02-23 15:35:38 +09001423 t->expire = tick_add(now_ms, MS_TO_TICKS(check->inter));
Krzysztof Piotr Oledzki09605412009-09-23 22:09:24 +02001424
Willy Tarreaufb56aab2012-09-28 14:40:02 +02001425 if (s->proxy->timeout.check && s->proxy->timeout.connect) {
1426 int t_con = tick_add(now_ms, s->proxy->timeout.connect);
1427 t->expire = tick_first(t->expire, t_con);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001428 }
Willy Tarreaudb3b4a22012-11-23 16:32:33 +01001429 conn_data_poll_recv(conn); /* prepare for reading a possible reply */
Willy Tarreau5a78f362012-11-23 12:47:05 +01001430 goto reschedule;
1431
1432 case SN_ERR_SRVTO: /* ETIMEDOUT */
1433 case SN_ERR_SRVCL: /* ECONNREFUSED, ENETUNREACH, ... */
Simon Horman4a741432013-02-23 15:35:38 +09001434 set_server_check_status(check, HCHK_STATUS_L4CON, strerror(errno));
Willy Tarreau5a78f362012-11-23 12:47:05 +01001435 break;
1436 case SN_ERR_PRXCOND:
1437 case SN_ERR_RESOURCE:
1438 case SN_ERR_INTERNAL:
Simon Horman4a741432013-02-23 15:35:38 +09001439 set_server_check_status(check, HCHK_STATUS_SOCKERR, NULL);
Willy Tarreau5a78f362012-11-23 12:47:05 +01001440 break;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001441 }
1442
Willy Tarreau5a78f362012-11-23 12:47:05 +01001443 /* here, we have seen a synchronous error, no fd was allocated */
Willy Tarreau6b0a8502012-11-23 08:51:32 +01001444
Simon Hormancd5d7b62013-02-24 17:23:38 +09001445 check->state &= ~CHK_STATE_RUNNING;
Simon Hormand8583062013-11-25 10:46:33 +09001446 check_failed(check);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001447
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001448 /* we allow up to min(inter, timeout.connect) for a connection
1449 * to establish but only when timeout.check is set
1450 * as it may be to short for a full check otherwise
1451 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001452 while (tick_is_expired(t->expire, now_ms)) {
1453 int t_con;
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001454
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001455 t_con = tick_add(t->expire, s->proxy->timeout.connect);
Simon Horman4a741432013-02-23 15:35:38 +09001456 t->expire = tick_add(t->expire, MS_TO_TICKS(check->inter));
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001457
Willy Tarreau0c303ee2008-07-07 00:09:58 +02001458 if (s->proxy->timeout.check)
1459 t->expire = tick_first(t->expire, t_con);
Krzysztof Piotr Oledzki5259dfe2008-01-21 01:54:06 +01001460 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001461 }
1462 else {
Willy Tarreauf1503172012-09-28 19:39:36 +02001463 /* there was a test running.
1464 * First, let's check whether there was an uncaught error,
1465 * which can happen on connect timeout or error.
1466 */
Simon Horman28b5ffc2013-02-24 07:25:29 +09001467 if (s->check.result == SRV_CHK_UNKNOWN) {
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001468 if ((conn->flags & (CO_FL_CONNECTED|CO_FL_WAIT_L4_CONN)) == CO_FL_WAIT_L4_CONN) {
1469 /* L4 not established (yet) */
Willy Tarreauf1503172012-09-28 19:39:36 +02001470 if (conn->flags & CO_FL_ERROR)
Simon Horman4a741432013-02-23 15:35:38 +09001471 set_server_check_status(check, HCHK_STATUS_L4CON, NULL);
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001472 else if (expired)
Simon Horman4a741432013-02-23 15:35:38 +09001473 set_server_check_status(check, HCHK_STATUS_L4TOUT, NULL);
Willy Tarreauf1503172012-09-28 19:39:36 +02001474 }
Willy Tarreau5a78f362012-11-23 12:47:05 +01001475 else if ((conn->flags & (CO_FL_CONNECTED|CO_FL_WAIT_L6_CONN)) == CO_FL_WAIT_L6_CONN) {
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001476 /* L6 not established (yet) */
Willy Tarreauf1503172012-09-28 19:39:36 +02001477 if (conn->flags & CO_FL_ERROR)
Simon Horman4a741432013-02-23 15:35:38 +09001478 set_server_check_status(check, HCHK_STATUS_L6RSP, NULL);
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001479 else if (expired)
Simon Horman4a741432013-02-23 15:35:38 +09001480 set_server_check_status(check, HCHK_STATUS_L6TOUT, NULL);
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001481 }
Simon Horman4a741432013-02-23 15:35:38 +09001482 else if (!check->type) {
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001483 /* good connection is enough for pure TCP check */
Simon Horman4a741432013-02-23 15:35:38 +09001484 if (check->use_ssl)
1485 set_server_check_status(check, HCHK_STATUS_L6OK, NULL);
Willy Tarreauf1503172012-09-28 19:39:36 +02001486 else
Simon Horman4a741432013-02-23 15:35:38 +09001487 set_server_check_status(check, HCHK_STATUS_L4OK, NULL);
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001488 }
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001489 else if (expired) {
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001490 /* connection established but expired check */
Simon Horman4a741432013-02-23 15:35:38 +09001491 if (check->type == PR_O2_SSL3_CHK)
1492 set_server_check_status(check, HCHK_STATUS_L6TOUT, NULL);
Willy Tarreauacbdc7a2012-11-23 14:02:10 +01001493 else /* HTTP, SMTP, ... */
Simon Horman4a741432013-02-23 15:35:38 +09001494 set_server_check_status(check, HCHK_STATUS_L7TOUT, NULL);
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001495
Willy Tarreauf1503172012-09-28 19:39:36 +02001496 }
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001497 else
1498 goto out_wait; /* timeout not reached, wait again */
Willy Tarreauf1503172012-09-28 19:39:36 +02001499 }
1500
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001501 /* check complete or aborted */
Willy Tarreau5ba04f62013-02-12 15:23:12 +01001502 if (conn->xprt) {
1503 /* The check was aborted and the connection was not yet closed.
1504 * This can happen upon timeout, or when an external event such
1505 * as a failed response coupled with "observe layer7" caused the
1506 * server state to be suddenly changed.
1507 */
1508 if (conn->ctrl)
1509 setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
1510 (struct linger *) &nolinger, sizeof(struct linger));
1511 conn_full_close(conn);
1512 }
1513
Simon Hormand8583062013-11-25 10:46:33 +09001514 if (check->result & SRV_CHK_FAILED) /* a failure or timeout detected */
1515 check_failed(check);
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001516 else { /* check was OK */
Willy Tarreau48494c02007-11-30 10:41:39 +01001517 /* we may have to add/remove this server from the LB group */
1518 if ((s->state & SRV_RUNNING) && (s->proxy->options & PR_O_DISABLE404)) {
Simon Horman4a741432013-02-23 15:35:38 +09001519 if ((s->state & SRV_GOINGDOWN) && !(check->result & SRV_CHK_DISABLE))
1520 set_server_enabled(check);
1521 else if (!(s->state & SRV_GOINGDOWN) && (check->result & SRV_CHK_DISABLE))
1522 set_server_disabled(check);
Willy Tarreau48494c02007-11-30 10:41:39 +01001523 }
1524
Simon Horman58c32972013-11-25 10:46:38 +09001525 if (check->health < check->rise + check->fall - 1) {
Simon Horman125d0992013-02-24 17:23:38 +09001526 check->health++; /* was bad, stays for a while */
Simon Horman4a741432013-02-23 15:35:38 +09001527 set_server_up(check);
Willy Tarreaubaaee002006-06-26 02:48:02 +02001528 }
Willy Tarreaubaaee002006-06-26 02:48:02 +02001529 }
Simon Hormancd5d7b62013-02-24 17:23:38 +09001530 check->state &= ~CHK_STATE_RUNNING;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001531
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001532 rv = 0;
1533 if (global.spread_checks > 0) {
Simon Horman4a741432013-02-23 15:35:38 +09001534 rv = srv_getinter(check) * global.spread_checks / 100;
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001535 rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0)));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001536 }
Simon Horman4a741432013-02-23 15:35:38 +09001537 t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(check) + rv));
Willy Tarreaubaaee002006-06-26 02:48:02 +02001538 }
Willy Tarreau5a78f362012-11-23 12:47:05 +01001539
1540 reschedule:
1541 while (tick_is_expired(t->expire, now_ms))
Simon Horman4a741432013-02-23 15:35:38 +09001542 t->expire = tick_add(t->expire, MS_TO_TICKS(check->inter));
Willy Tarreau74fa7fb2012-11-23 14:43:49 +01001543 out_wait:
Willy Tarreau26c25062009-03-08 09:38:41 +01001544 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +02001545}
1546
Simon Horman5c942422013-11-25 10:46:32 +09001547static int start_check_task(struct check *check, int mininter,
1548 int nbcheck, int srvpos)
1549{
1550 struct task *t;
1551 /* task for the check */
1552 if ((t = task_new()) == NULL) {
1553 Alert("Starting [%s:%s] check: out of memory.\n",
1554 check->server->proxy->id, check->server->id);
1555 return 0;
1556 }
1557
1558 check->task = t;
1559 t->process = process_chk;
1560 t->context = check;
1561
1562 /* check this every ms */
1563 t->expire = tick_add(now_ms,
1564 MS_TO_TICKS(((mininter &&
1565 mininter >= srv_getinter(check)) ?
1566 mininter : srv_getinter(check)) * srvpos / nbcheck));
1567 check->start = now;
1568 task_queue(t);
1569
1570 return 1;
1571}
1572
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001573/*
1574 * Start health-check.
1575 * Returns 0 if OK, -1 if error, and prints the error in this case.
1576 */
1577int start_checks() {
1578
1579 struct proxy *px;
1580 struct server *s;
1581 struct task *t;
Simon Horman4a741432013-02-23 15:35:38 +09001582 int nbcheck=0, mininter=0, srvpos=0;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001583
Willy Tarreau2c43a1e2007-10-14 23:05:39 +02001584 /* 1- count the checkers to run simultaneously.
1585 * We also determine the minimum interval among all of those which
1586 * have an interval larger than SRV_CHK_INTER_THRES. This interval
1587 * will be used to spread their start-up date. Those which have
Jamie Gloudon801a0a32012-08-25 00:18:33 -04001588 * a shorter interval will start independently and will not dictate
Willy Tarreau2c43a1e2007-10-14 23:05:39 +02001589 * too short an interval for all others.
1590 */
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001591 for (px = proxy; px; px = px->next) {
1592 for (s = px->srv; s; s = s->next) {
Willy Tarreaue7b73482013-11-21 11:50:50 +01001593 if (s->slowstart) {
1594 if ((t = task_new()) == NULL) {
1595 Alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id);
1596 return -1;
1597 }
1598 /* We need a warmup task that will be called when the server
1599 * state switches from down to up.
1600 */
1601 s->warmup = t;
1602 t->process = server_warmup;
1603 t->context = s;
1604 t->expire = TICK_ETERNITY;
1605 }
1606
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001607 if (!(s->state & SRV_CHECKED))
1608 continue;
1609
Simon Horman4a741432013-02-23 15:35:38 +09001610 if ((srv_getinter(&s->check) >= SRV_CHK_INTER_THRES) &&
1611 (!mininter || mininter > srv_getinter(&s->check)))
1612 mininter = srv_getinter(&s->check);
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001613
Simon Horman4a741432013-02-23 15:35:38 +09001614 nbcheck++;
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001615 }
1616 }
1617
Simon Horman4a741432013-02-23 15:35:38 +09001618 if (!nbcheck)
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001619 return 0;
1620
1621 srand((unsigned)time(NULL));
1622
1623 /*
1624 * 2- start them as far as possible from each others. For this, we will
1625 * start them after their interval set to the min interval divided by
1626 * the number of servers, weighted by the server's position in the list.
1627 */
1628 for (px = proxy; px; px = px->next) {
1629 for (s = px->srv; s; s = s->next) {
Simon Hormand60d6912013-11-25 10:46:36 +09001630 /* A task for the main check */
1631 if (s->state & SRV_CHECKED) {
1632 if (!start_check_task(&s->check, mininter, nbcheck, srvpos))
1633 return -1;
1634 srvpos++;
1635 }
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001636
Simon Hormand60d6912013-11-25 10:46:36 +09001637 /* A task for a auxiliary agent check */
1638 if (s->state & SRV_AGENT_CHECKED) {
1639 if (!start_check_task(&s->agent, mininter, nbcheck, srvpos)) {
1640 return -1;
1641 }
1642 srvpos++;
1643 }
Krzysztof Oledzkib304dc72007-10-14 23:40:01 +02001644 }
1645 }
1646 return 0;
1647}
Willy Tarreaubaaee002006-06-26 02:48:02 +02001648
1649/*
Willy Tarreau5b3a2022012-09-28 15:01:02 +02001650 * Perform content verification check on data in s->check.buffer buffer.
Willy Tarreaubd741542010-03-16 18:46:54 +01001651 * The buffer MUST be terminated by a null byte before calling this function.
1652 * Sets server status appropriately. The caller is responsible for ensuring
1653 * that the buffer contains at least 13 characters. If <done> is zero, we may
1654 * return 0 to indicate that data is required to decide of a match.
1655 */
1656static int httpchk_expect(struct server *s, int done)
1657{
1658 static char status_msg[] = "HTTP status check returned code <000>";
1659 char status_code[] = "000";
1660 char *contentptr;
1661 int crlf;
1662 int ret;
1663
1664 switch (s->proxy->options2 & PR_O2_EXP_TYPE) {
1665 case PR_O2_EXP_STS:
1666 case PR_O2_EXP_RSTS:
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001667 memcpy(status_code, s->check.bi->data + 9, 3);
1668 memcpy(status_msg + strlen(status_msg) - 4, s->check.bi->data + 9, 3);
Willy Tarreaubd741542010-03-16 18:46:54 +01001669
1670 if ((s->proxy->options2 & PR_O2_EXP_TYPE) == PR_O2_EXP_STS)
1671 ret = strncmp(s->proxy->expect_str, status_code, 3) == 0;
1672 else
1673 ret = regexec(s->proxy->expect_regex, status_code, MAX_MATCH, pmatch, 0) == 0;
1674
1675 /* we necessarily have the response, so there are no partial failures */
1676 if (s->proxy->options2 & PR_O2_EXP_INV)
1677 ret = !ret;
1678
Simon Horman4a741432013-02-23 15:35:38 +09001679 set_server_check_status(&s->check, ret ? HCHK_STATUS_L7OKD : HCHK_STATUS_L7STS, status_msg);
Willy Tarreaubd741542010-03-16 18:46:54 +01001680 break;
1681
1682 case PR_O2_EXP_STR:
1683 case PR_O2_EXP_RSTR:
1684 /* very simple response parser: ignore CR and only count consecutive LFs,
1685 * stop with contentptr pointing to first char after the double CRLF or
1686 * to '\0' if crlf < 2.
1687 */
1688 crlf = 0;
Willy Tarreau1ae1b7b2012-09-28 15:28:30 +02001689 for (contentptr = s->check.bi->data; *contentptr; contentptr++) {
Willy Tarreaubd741542010-03-16 18:46:54 +01001690 if (crlf >= 2)
1691 break;
1692 if (*contentptr == '\r')
1693 continue;
1694 else if (*contentptr == '\n')
1695 crlf++;
1696 else
1697 crlf = 0;
1698 }
1699
1700 /* Check that response contains a body... */
1701 if (crlf < 2) {
1702 if (!done)
1703 return 0;
1704
Simon Horman4a741432013-02-23 15:35:38 +09001705 set_server_check_status(&s->check, HCHK_STATUS_L7RSP,
Willy Tarreaubd741542010-03-16 18:46:54 +01001706 "HTTP content check could not find a response body");
1707 return 1;
1708 }
1709
1710 /* Check that response body is not empty... */
1711 if (*contentptr == '\0') {
Willy Tarreaua164fb52011-04-13 09:32:41 +02001712 if (!done)
1713 return 0;
1714
Simon Horman4a741432013-02-23 15:35:38 +09001715 set_server_check_status(&s->check, HCHK_STATUS_L7RSP,
Willy Tarreaubd741542010-03-16 18:46:54 +01001716 "HTTP content check found empty response body");
1717 return 1;
1718 }
1719
1720 /* Check the response content against the supplied string
1721 * or regex... */
1722 if ((s->proxy->options2 & PR_O2_EXP_TYPE) == PR_O2_EXP_STR)
1723 ret = strstr(contentptr, s->proxy->expect_str) != NULL;
1724 else
1725 ret = regexec(s->proxy->expect_regex, contentptr, MAX_MATCH, pmatch, 0) == 0;
1726
1727 /* if we don't match, we may need to wait more */
1728 if (!ret && !done)
1729 return 0;
1730
1731 if (ret) {
1732 /* content matched */
1733 if (s->proxy->options2 & PR_O2_EXP_INV)
Simon Horman4a741432013-02-23 15:35:38 +09001734 set_server_check_status(&s->check, HCHK_STATUS_L7RSP,
Willy Tarreaubd741542010-03-16 18:46:54 +01001735 "HTTP check matched unwanted content");
1736 else
Simon Horman4a741432013-02-23 15:35:38 +09001737 set_server_check_status(&s->check, HCHK_STATUS_L7OKD,
Willy Tarreaubd741542010-03-16 18:46:54 +01001738 "HTTP content check matched");
1739 }
1740 else {
1741 if (s->proxy->options2 & PR_O2_EXP_INV)
Simon Horman4a741432013-02-23 15:35:38 +09001742 set_server_check_status(&s->check, HCHK_STATUS_L7OKD,
Willy Tarreaubd741542010-03-16 18:46:54 +01001743 "HTTP check did not match unwanted content");
1744 else
Simon Horman4a741432013-02-23 15:35:38 +09001745 set_server_check_status(&s->check, HCHK_STATUS_L7RSP,
Willy Tarreaubd741542010-03-16 18:46:54 +01001746 "HTTP content check did not match");
1747 }
1748 break;
1749 }
1750 return 1;
1751}
1752
1753/*
Willy Tarreaubaaee002006-06-26 02:48:02 +02001754 * Local variables:
1755 * c-indent-level: 8
1756 * c-basic-offset: 8
1757 * End:
1758 */