[MEDIUM] add a maintenance mode to servers
This is a first attempt to add a maintenance mode on servers, using
the stat socket (in admin level).
It can be done with the following command :
- disable server <backend>/<server>
- enable server <backend>/<server>
In this mode, no more checks will be performed on the server and it
will be marked as a special DOWN state (MAINT).
If some servers were tracking it, they'll go DOWN until the server
leaves the maintenance mode. The stats page and the CSV export also
display this special state.
This can be used to disable the server in haproxy before doing some
operations on this server itself. This is a good complement to the
"http-check disable-on-404" keyword and works in TCP mode.
diff --git a/src/checks.c b/src/checks.c
index f59fbbe..b235d9e 100644
--- a/src/checks.c
+++ b/src/checks.c
@@ -359,12 +359,16 @@
* possible to other servers. It automatically recomputes the number of
* servers, but not the map.
*/
-static void set_server_down(struct server *s)
+void set_server_down(struct server *s)
{
struct server *srv;
struct chunk msg;
int xferred;
+ if (s->state & SRV_MAINTAIN) {
+ s->health = s->rise;
+ }
+
if (s->health == s->rise || s->tracked) {
int srv_was_paused = s->state & SRV_GOINGDOWN;
@@ -380,14 +384,19 @@
chunk_init(&msg, trash, sizeof(trash));
- chunk_printf(&msg,
- "%sServer %s/%s is DOWN", s->state & SRV_BACKUP ? "Backup " : "",
- s->proxy->id, s->id);
+ if (s->state & SRV_MAINTAIN) {
+ chunk_printf(&msg,
+ "%sServer %s/%s is DOWN for maintenance", s->state & SRV_BACKUP ? "Backup " : "",
+ s->proxy->id, s->id);
+ } else {
+ chunk_printf(&msg,
+ "%sServer %s/%s is DOWN", s->state & SRV_BACKUP ? "Backup " : "",
+ s->proxy->id, s->id);
- server_status_printf(&msg, s,
- ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
- xferred);
-
+ server_status_printf(&msg, s,
+ ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
+ xferred);
+ }
Warning("%s.\n", trash);
/* we don't send an alert if the server was previously paused */
@@ -403,18 +412,24 @@
if (s->state & SRV_CHECKED)
for(srv = s->tracknext; srv; srv = srv->tracknext)
- set_server_down(srv);
+ if (! (srv->state & SRV_MAINTAIN))
+ /* Only notify tracking servers that are not already in maintenance. */
+ set_server_down(srv);
}
s->health = 0; /* failure */
}
-static void set_server_up(struct server *s) {
+void set_server_up(struct server *s) {
struct server *srv;
struct chunk msg;
int xferred;
+ if (s->state & SRV_MAINTAIN) {
+ s->health = s->rise;
+ }
+
if (s->health == s->rise || s->tracked) {
if (s->proxy->srv_bck == 0 && s->proxy->srv_act == 0) {
if (s->proxy->last_change < now.tv_sec) // ignore negative times
@@ -448,20 +463,30 @@
chunk_init(&msg, trash, sizeof(trash));
- chunk_printf(&msg,
- "%sServer %s/%s is UP", s->state & SRV_BACKUP ? "Backup " : "",
- s->proxy->id, s->id);
+ if (s->state & SRV_MAINTAIN) {
+ chunk_printf(&msg,
+ "%sServer %s/%s is UP (leaving maintenance)", s->state & SRV_BACKUP ? "Backup " : "",
+ s->proxy->id, s->id);
+ } else {
+ chunk_printf(&msg,
+ "%sServer %s/%s is UP", s->state & SRV_BACKUP ? "Backup " : "",
+ s->proxy->id, s->id);
- server_status_printf(&msg, s,
- ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
- xferred);
+ server_status_printf(&msg, s,
+ ((!s->tracked && !(s->proxy->options2 & PR_O2_LOGHCHKS))?SSP_O_HCHK:0),
+ xferred);
+ }
Warning("%s.\n", trash);
send_log(s->proxy, LOG_NOTICE, "%s.\n", trash);
if (s->state & SRV_CHECKED)
for(srv = s->tracknext; srv; srv = srv->tracknext)
- set_server_up(srv);
+ if (! (srv->state & SRV_MAINTAIN))
+ /* Only notify tracking servers if they're not in maintenance. */
+ set_server_up(srv);
+
+ s->state &= ~SRV_MAINTAIN;
}
if (s->health >= s->rise)
@@ -1007,7 +1032,7 @@
/* we don't send any health-checks when the proxy is stopped or when
* the server should not be checked.
*/
- if (!(s->state & SRV_CHECKED) || s->proxy->state == PR_STSTOPPED) {
+ if (!(s->state & SRV_CHECKED) || s->proxy->state == PR_STSTOPPED || (s->state & SRV_MAINTAIN)) {
while (tick_is_expired(t->expire, now_ms))
t->expire = tick_add(t->expire, MS_TO_TICKS(s->inter));
return t;
diff --git a/src/dumpstats.c b/src/dumpstats.c
index 24b0706..8900636 100644
--- a/src/dumpstats.c
+++ b/src/dumpstats.c
@@ -65,6 +65,8 @@
" get weight : report a server's current weight\n"
" set weight : change a server's weight\n"
" set timeout : change a timeout setting\n"
+ " disable server : set a server in maintenance mode\n"
+ " enable server : re-enable a server that was previously in maintenance mode\n"
"";
const char stats_permission_denied_msg[] =
@@ -529,7 +531,102 @@
return 0;
}
}
- else { /* not "show" nor "clear" nor "get" nor "set" */
+ else if (strcmp(args[0], "enable") == 0) {
+ if (strcmp(args[1], "server") == 0) {
+ struct proxy *px;
+ struct server *sv;
+
+ if (s->listener->perm.ux.level < ACCESS_LVL_ADMIN) {
+ s->data_ctx.cli.msg = stats_permission_denied_msg;
+ si->st0 = STAT_CLI_PRINT;
+ return 1;
+ }
+
+ /* split "backend/server" and make <line> point to server */
+ for (line = args[2]; *line; line++)
+ if (*line == '/') {
+ *line++ = '\0';
+ break;
+ }
+
+ if (!*line || !*args[2]) {
+ s->data_ctx.cli.msg = "Require 'backend/server'.\n";
+ si->st0 = STAT_CLI_PRINT;
+ return 1;
+ }
+
+ if (!get_backend_server(args[2], line, &px, &sv)) {
+ s->data_ctx.cli.msg = px ? "No such server.\n" : "No such backend.\n";
+ si->st0 = STAT_CLI_PRINT;
+ return 1;
+ }
+
+ if (sv->state & SRV_MAINTAIN) {
+ /* The server is really in maintenance, we can change the server state */
+ if (sv->tracked) {
+ /* If this server tracks the status of another one,
+ * we must restore the good status.
+ */
+ if (sv->tracked->state & SRV_RUNNING) {
+ set_server_up(sv);
+ } else {
+ sv->state &= ~SRV_MAINTAIN;
+ set_server_down(sv);
+ }
+ } else {
+ set_server_up(sv);
+ }
+ }
+
+ return 1;
+ }
+ else { /* unknown "enable" parameter */
+ return 0;
+ }
+ }
+ else if (strcmp(args[0], "disable") == 0) {
+ if (strcmp(args[1], "server") == 0) {
+ struct proxy *px;
+ struct server *sv;
+
+ if (s->listener->perm.ux.level < ACCESS_LVL_ADMIN) {
+ s->data_ctx.cli.msg = stats_permission_denied_msg;
+ si->st0 = STAT_CLI_PRINT;
+ return 1;
+ }
+
+ /* split "backend/server" and make <line> point to server */
+ for (line = args[2]; *line; line++)
+ if (*line == '/') {
+ *line++ = '\0';
+ break;
+ }
+
+ if (!*line || !*args[2]) {
+ s->data_ctx.cli.msg = "Require 'backend/server'.\n";
+ si->st0 = STAT_CLI_PRINT;
+ return 1;
+ }
+
+ if (!get_backend_server(args[2], line, &px, &sv)) {
+ s->data_ctx.cli.msg = px ? "No such server.\n" : "No such backend.\n";
+ si->st0 = STAT_CLI_PRINT;
+ return 1;
+ }
+
+ if (! (sv->state & SRV_MAINTAIN)) {
+ /* Not already in maintenance, we can change the server state */
+ sv->state |= SRV_MAINTAIN;
+ set_server_down(sv);
+ }
+
+ return 1;
+ }
+ else { /* unknown "disable" parameter */
+ return 0;
+ }
+ }
+ else { /* not "show" nor "clear" nor "get" nor "set" nor "enable" nor "disable" */
return 0;
}
return 1;
@@ -1015,6 +1112,7 @@
".backup4 {background: #c060ff;}\n" /* NOLB state shows same as going down */
".backup5 {background: #90b0e0;}\n" /* NOLB state shows same as going down */
".backup6 {background: #e0e0e0;}\n"
+ ".maintain {background: #c07820;}\n"
".rls {letter-spacing: 0.2em; margin-right: 1px;}\n" /* right letter spacing (used for grouping digits) */
"\n"
"a.px:link {color: #ffff40; text-decoration: none;}"
@@ -1083,6 +1181,8 @@
"</tr><tr>\n"
"<td class=\"active0\"></td><td class=\"noborder\">active or backup DOWN </td>"
"<td class=\"active6\"></td><td class=\"noborder\">not checked </td>"
+ "</tr><tr>\n"
+ "<td class=\"maintain\"></td><td class=\"noborder\" colspan=\"3\">active or backup DOWN for maintenance (MAINT) </td>"
"</tr></table>\n"
"Note: UP with load-balancing disabled is reported as \"NOLB\"."
"</td>"
@@ -1608,10 +1708,18 @@
"UP %d/%d ↓", "UP",
"NOLB %d/%d ↓", "NOLB",
"<i>no check</i>" };
- chunk_printf(&msg,
- /* name */
- "<tr class=\"%s%d\"><td class=ac",
- (sv->state & SRV_BACKUP) ? "backup" : "active", sv_state);
+ if (sv->state & SRV_MAINTAIN) {
+ chunk_printf(&msg,
+ /* name */
+ "<tr class=\"maintain\"><td class=ac"
+ );
+ }
+ else {
+ chunk_printf(&msg,
+ /* name */
+ "<tr class=\"%s%d\"><td class=ac",
+ (sv->state & SRV_BACKUP) ? "backup" : "active", sv_state);
+ }
if (uri->flags&ST_SHLGNDS) {
char str[INET6_ADDRSTRLEN];
@@ -1693,7 +1801,12 @@
/* status, lest check */
chunk_printf(&msg, "<td class=ac>");
- if (svs->state & SRV_CHECKED) {
+ if (sv->state & SRV_MAINTAIN) {
+ chunk_printf(&msg, "%s ",
+ human_time(now.tv_sec - sv->last_change, 1));
+ chunk_printf(&msg, "MAINT");
+ }
+ else if (svs->state & SRV_CHECKED) {
chunk_printf(&msg, "%s ",
human_time(now.tv_sec - sv->last_change, 1));
@@ -1801,10 +1914,14 @@
sv->counters.retries, sv->counters.redispatches);
/* status */
- chunk_printf(&msg,
- srv_hlt_st[sv_state],
- (sv->state & SRV_RUNNING) ? (sv->health - sv->rise + 1) : (sv->health),
- (sv->state & SRV_RUNNING) ? (sv->fall) : (sv->rise));
+ if (sv->state & SRV_MAINTAIN) {
+ chunk_printf(&msg, "MAINT,");
+ } else {
+ chunk_printf(&msg,
+ srv_hlt_st[sv_state],
+ (svs->state & SRV_RUNNING) ? (svs->health - svs->rise + 1) : (svs->health),
+ (svs->state & SRV_RUNNING) ? (svs->fall) : (svs->rise));
+ }
chunk_printf(&msg,
/* weight, active, backup */