MINOR: proxy: Don't close FDs if not our proxy.
When running with multiple process, if some proxies are just assigned
to some processes, the other processes will just close the file descriptors
for the listening sockets. However, we may still have to provide those
sockets when reloading, so instead we just try hard to pretend those proxies
are dead, while keeping the sockets opened.
A new global option, no-reused-socket", has been added, to restore the old
behavior of closing the sockets not bound to this process.
diff --git a/doc/configuration.txt b/doc/configuration.txt
index 05f0701..e6ea2cf 100644
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -587,6 +587,7 @@
- nosplice
- nogetaddrinfo
- noreuseport
+ - no-unused-socket
- spread-checks
- server-state-base
- server-state-file
@@ -1250,6 +1251,12 @@
Disables the use of SO_REUSEPORT - see socket(7). It is equivalent to the
command line argument "-dR".
+no-unused-socket
+ By default, each haproxy process keeps all sockets opened, event those that
+ are only used by another processes, so that any process can provide all the
+ sockets, to make reloads seamless. This option disables this, and close all
+ unused sockets, to save some file descriptors.
+
spread-checks <0..50, in percent>
Sometimes it is desirable to avoid sending agent and health checks to
servers at exact intervals, for instance when many logical servers are
diff --git a/include/proto/fd.h b/include/proto/fd.h
index 87309bf..1efe323 100644
--- a/include/proto/fd.h
+++ b/include/proto/fd.h
@@ -41,6 +41,11 @@
*/
void fd_delete(int fd);
+/* Deletes an FD from the fdsets, and recomputes the maxfd limit.
+ * The file descriptor is kept open.
+ */
+void fd_remove(int fd);
+
/* disable the specified poller */
void disable_poller(const char *poller_name);
diff --git a/include/proto/listener.h b/include/proto/listener.h
index 552d695..079d976 100644
--- a/include/proto/listener.h
+++ b/include/proto/listener.h
@@ -88,6 +88,11 @@
*/
int unbind_listener(struct listener *listener);
+/* This function pretends the listener is dead, but keeps the FD opened, so
+ * that we can provide it, for conf reloading.
+ */
+int unbind_listener_no_close(struct listener *listener);
+
/* This function closes all listening sockets bound to the protocol <proto>,
* and the listeners end in LI_ASSIGNED state if they were higher. It does not
* detach them from the protocol. It always returns ERR_NONE.
diff --git a/include/proto/proxy.h b/include/proto/proxy.h
index 72f1e1d..e5d20c4 100644
--- a/include/proto/proxy.h
+++ b/include/proto/proxy.h
@@ -42,6 +42,7 @@
int pause_proxy(struct proxy *p);
int resume_proxy(struct proxy *p);
void stop_proxy(struct proxy *p);
+void zombify_proxy(struct proxy *p);
void pause_proxies(void);
void resume_proxies(void);
int stream_set_backend(struct stream *s, struct proxy *be);
diff --git a/include/types/global.h b/include/types/global.h
index df8e2c6..57b969d 100644
--- a/include/types/global.h
+++ b/include/types/global.h
@@ -62,6 +62,8 @@
#define GTUNE_USE_REUSEPORT (1<<6)
#define GTUNE_RESOLVE_DONTFAIL (1<<7)
+#define GTUNE_SOCKET_TRANSFER (1<<8)
+
/* Access level for a stats socket */
#define ACCESS_LVL_NONE 0
#define ACCESS_LVL_USER 1
diff --git a/include/types/listener.h b/include/types/listener.h
index 227cc28..2b8f5fe 100644
--- a/include/types/listener.h
+++ b/include/types/listener.h
@@ -47,6 +47,7 @@
LI_INIT, /* all parameters filled in, but not assigned yet */
LI_ASSIGNED, /* assigned to the protocol, but not listening yet */
LI_PAUSED, /* listener was paused, it's bound but not listening */
+ LI_ZOMBIE, /* The listener doesn't belong to the process, but is kept opened */
LI_LISTEN, /* started, listening but not enabled */
LI_READY, /* started, listening and enabled */
LI_FULL, /* reached its connection limit */
diff --git a/src/cfgparse.c b/src/cfgparse.c
index 47d33cf..348b9e8 100644
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -659,6 +659,11 @@
goto out;
global.tune.options &= ~GTUNE_USE_REUSEPORT;
}
+ else if (!strcmp(args[0], "no-unused-socket")) {
+ if (alertif_too_many_args(0, file, linenum, args, &err_code))
+ goto out;
+ global.tune.options &= ~GTUNE_SOCKET_TRANSFER;
+ }
else if (!strcmp(args[0], "quiet")) {
if (alertif_too_many_args(0, file, linenum, args, &err_code))
goto out;
diff --git a/src/cli.c b/src/cli.c
index 54fb438..20e143b 100644
--- a/src/cli.c
+++ b/src/cli.c
@@ -1063,10 +1063,11 @@
struct listener *l;
list_for_each_entry(l, &px->conf.listeners, by_fe) {
- /* Only transfer IPv4/IPv6 sockets */
- if (l->proto->sock_family == AF_INET ||
+ /* Only transfer IPv4/IPv6/UNIX sockets */
+ if (l->state >= LI_ZOMBIE &&
+ (l->proto->sock_family == AF_INET ||
l->proto->sock_family == AF_INET6 ||
- l->proto->sock_family == AF_UNIX)
+ l->proto->sock_family == AF_UNIX))
tot_fd_nb++;
}
px = px->next;
@@ -1117,7 +1118,7 @@
list_for_each_entry(l, &px->conf.listeners, by_fe) {
int ret;
/* Only transfer IPv4/IPv6 sockets */
- if (l->state >= LI_LISTEN &&
+ if (l->state >= LI_ZOMBIE &&
(l->proto->sock_family == AF_INET ||
l->proto->sock_family == AF_INET6 ||
l->proto->sock_family == AF_UNIX)) {
diff --git a/src/fd.c b/src/fd.c
index aeee602..1a62f9a 100644
--- a/src/fd.c
+++ b/src/fd.c
@@ -175,7 +175,7 @@
/* Deletes an FD from the fdsets, and recomputes the maxfd limit.
* The file descriptor is also closed.
*/
-void fd_delete(int fd)
+static void fd_dodelete(int fd, int do_close)
{
if (fdtab[fd].linger_risk) {
/* this is generally set when connecting to servers */
@@ -190,7 +190,8 @@
port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
fdinfo[fd].port_range = NULL;
- close(fd);
+ if (do_close)
+ close(fd);
fdtab[fd].owner = NULL;
fdtab[fd].new = 0;
@@ -198,6 +199,22 @@
maxfd--;
}
+/* Deletes an FD from the fdsets, and recomputes the maxfd limit.
+ * The file descriptor is also closed.
+ */
+void fd_delete(int fd)
+{
+ fd_dodelete(fd, 1);
+}
+
+/* Deletes an FD from the fdsets, and recomputes the maxfd limit.
+ * The file descriptor is kept open.
+ */
+void fd_remove(int fd)
+{
+ fd_dodelete(fd, 0);
+}
+
/* Scan and process the cached events. This should be called right after
* the poller. The loop may cause new entries to be created, for example
* if a listener causes an accept() to initiate a new incoming connection
diff --git a/src/haproxy.c b/src/haproxy.c
index bf176df..01969c9 100644
--- a/src/haproxy.c
+++ b/src/haproxy.c
@@ -857,6 +857,7 @@
#if defined(SO_REUSEPORT)
global.tune.options |= GTUNE_USE_REUSEPORT;
#endif
+ global.tune.options |= GTUNE_SOCKET_TRANSFER;
pid = getpid();
progname = *argv;
@@ -1668,6 +1669,15 @@
}/* end while(s) */
list_for_each_entry_safe(l, l_next, &p->conf.listeners, by_fe) {
+ /*
+ * Zombie proxy, the listener just pretend to be up
+ * because they still hold an opened fd.
+ * Close it and give the listener its real state.
+ */
+ if (p->state == PR_STSTOPPED && l->state >= LI_ZOMBIE) {
+ close(l->fd);
+ l->state = LI_INIT;
+ }
unbind_listener(l);
delete_listener(l);
LIST_DEL(&l->by_fe);
@@ -2148,8 +2158,12 @@
px = proxy;
while (px != NULL) {
if (px->bind_proc && px->state != PR_STSTOPPED) {
- if (!(px->bind_proc & (1UL << proc)))
- stop_proxy(px);
+ if (!(px->bind_proc & (1UL << proc))) {
+ if (global.tune.options & GTUNE_SOCKET_TRANSFER)
+ zombify_proxy(px);
+ else
+ stop_proxy(px);
+ }
}
px = px->next;
}
diff --git a/src/listener.c b/src/listener.c
index 7a1df0e..a99e4c0 100644
--- a/src/listener.c
+++ b/src/listener.c
@@ -59,7 +59,12 @@
/* we don't want to enable this listener and don't
* want any fd event to reach it.
*/
- unbind_listener(listener);
+ if (!(global.tune.options & GTUNE_SOCKET_TRANSFER))
+ unbind_listener(listener);
+ else {
+ unbind_listener_no_close(listener);
+ listener->state = LI_LISTEN;
+ }
}
else if (listener->nbconn < listener->maxconn) {
fd_want_recv(listener->fd);
@@ -95,7 +100,7 @@
*/
int pause_listener(struct listener *l)
{
- if (l->state <= LI_PAUSED)
+ if (l->state <= LI_ZOMBIE)
return 1;
if (l->proto->pause) {
@@ -149,7 +154,7 @@
return 0;
}
- if (l->state < LI_PAUSED)
+ if (l->state < LI_PAUSED || l->state == LI_ZOMBIE)
return 0;
if (l->proto->sock_prot == IPPROTO_TCP &&
@@ -242,12 +247,7 @@
}
}
-/* This function closes the listening socket for the specified listener,
- * provided that it's already in a listening state. The listener enters the
- * LI_ASSIGNED state. It always returns ERR_NONE. This function is intended
- * to be used as a generic function for standard protocols.
- */
-int unbind_listener(struct listener *listener)
+static int do_unbind_listener(struct listener *listener, int do_close)
{
if (listener->state == LI_READY)
fd_stop_recv(listener->fd);
@@ -256,13 +256,35 @@
LIST_DEL(&listener->wait_queue);
if (listener->state >= LI_PAUSED) {
- fd_delete(listener->fd);
- listener->fd = -1;
+ if (do_close) {
+ fd_delete(listener->fd);
+ listener->fd = -1;
+ }
+ else
+ fd_remove(listener->fd);
listener->state = LI_ASSIGNED;
}
return ERR_NONE;
}
+/* This function closes the listening socket for the specified listener,
+ * provided that it's already in a listening state. The listener enters the
+ * LI_ASSIGNED state. It always returns ERR_NONE. This function is intended
+ * to be used as a generic function for standard protocols.
+ */
+int unbind_listener(struct listener *listener)
+{
+ return do_unbind_listener(listener, 1);
+}
+
+/* This function pretends the listener is dead, but keeps the FD opened, so
+ * that we can provide it, for conf reloading.
+ */
+int unbind_listener_no_close(struct listener *listener)
+{
+ return do_unbind_listener(listener, 0);
+}
+
/* This function closes all listening sockets bound to the protocol <proto>,
* and the listeners end in LI_ASSIGNED state if they were higher. It does not
* detach them from the protocol. It always returns ERR_NONE.
diff --git a/src/proxy.c b/src/proxy.c
index d158fac..dc70213 100644
--- a/src/proxy.c
+++ b/src/proxy.c
@@ -991,6 +991,19 @@
p = proxy;
tv_update_date(0,1); /* else, the old time before select will be used */
while (p) {
+ /* Zombie proxy, let's close the file descriptors */
+ if (p->state == PR_STSTOPPED &&
+ !LIST_ISEMPTY(&p->conf.listeners) &&
+ LIST_ELEM(p->conf.listeners.n,
+ struct listener *, by_fe)->state >= LI_ZOMBIE) {
+ struct listener *l;
+ list_for_each_entry(l, &p->conf.listeners, by_fe) {
+ if (l->state >= LI_ZOMBIE)
+ close(l->fd);
+ l->state = LI_INIT;
+ }
+ }
+
if (p->state != PR_STSTOPPED) {
Warning("Stopping %s %s in %d ms.\n", proxy_cap_str(p->cap), p->id, p->grace);
send_log(p, LOG_WARNING, "Stopping %s %s in %d ms.\n", proxy_cap_str(p->cap), p->id, p->grace);
@@ -1051,6 +1064,45 @@
return 1;
}
+/* This function makes the proxy unusable, but keeps the listening sockets
+ * opened, so that if any process requests them, we are able to serve them.
+ * This should only be called early, before we started accepting requests.
+ */
+void zombify_proxy(struct proxy *p)
+{
+ struct listener *l;
+ struct listener *first_to_listen = NULL;
+
+ list_for_each_entry(l, &p->conf.listeners, by_fe) {
+ enum li_state oldstate = l->state;
+
+ unbind_listener_no_close(l);
+ if (l->state >= LI_ASSIGNED) {
+ delete_listener(l);
+ listeners--;
+ jobs--;
+ }
+ /*
+ * Pretend we're still up and running so that the fd
+ * will be sent if asked.
+ */
+ l->state = LI_ZOMBIE;
+ if (!first_to_listen && oldstate >= LI_LISTEN)
+ first_to_listen = l;
+ }
+ /* Quick hack : at stop time, to know we have to close the sockets
+ * despite the proxy being marked as stopped, make the first listener
+ * of the listener list an active one, so that we don't have to
+ * parse the whole list to be sure.
+ */
+ if (first_to_listen && LIST_ELEM(p->conf.listeners.n,
+ struct listener *, by_fe) != first_to_listen) {
+ LIST_DEL(&l->by_fe);
+ LIST_ADD(&p->conf.listeners, &l->by_fe);
+ }
+
+ p->state = PR_STSTOPPED;
+}
/*
* This function completely stops a proxy and releases its listeners. It has