BUG/MEDIUM: protocols: add a global lock for the init/deinit stuff
Dragan Dosen found that the listeners lock is not sufficient to protect
the listeners list when proxies are stopping because the listeners are
also unlinked from the protocol list, and under certain situations like
bombing with soft-stop signals or shutting down many frontends in parallel
from multiple CLI connections, it could be possible to provoke multiple
instances of delete_listener() to be called in parallel for different
listeners, thus corrupting the protocol lists.
Such operations are pretty rare, they are performed once per proxy upon
startup and once per proxy on shut down. Thus there is no point trying
to optimize anything and we can use a global lock to protect the protocol
lists during these manipulations.
This fix (or a variant) will have to be backported as far as 1.8.
(cherry picked from commit daacf3664506d56a1f3b050ccba504886a18b12a)
Signed-off-by: Christopher Faulet <cfaulet@haproxy.com>
diff --git a/include/proto/protocol.h b/include/proto/protocol.h
index 7bbebb8..f25f77f 100644
--- a/include/proto/protocol.h
+++ b/include/proto/protocol.h
@@ -23,9 +23,11 @@
#define _PROTO_PROTOCOL_H
#include <sys/socket.h>
+#include <common/hathreads.h>
#include <types/protocol.h>
extern struct protocol *__protocol_by_family[AF_CUST_MAX];
+__decl_hathreads(extern HA_SPINLOCK_T proto_lock);
/* Registers the protocol <proto> */
void protocol_register(struct protocol *proto);
diff --git a/include/types/protocol.h b/include/types/protocol.h
index 1d3404b..f38baeb 100644
--- a/include/types/protocol.h
+++ b/include/types/protocol.h
@@ -80,9 +80,9 @@
int (*pause)(struct listener *l); /* temporarily pause this listener for a soft restart */
void (*add)(struct listener *l, int port); /* add a listener for this protocol and port */
- struct list listeners; /* list of listeners using this protocol */
- int nb_listeners; /* number of listeners */
- struct list list; /* list of registered protocols */
+ struct list listeners; /* list of listeners using this protocol (under proto_lock) */
+ int nb_listeners; /* number of listeners (under proto_lock) */
+ struct list list; /* list of registered protocols (under proto_lock) */
};
#define CONNECT_HAS_DATA 0x00000001 /* There's data available to be sent */
diff --git a/src/listener.c b/src/listener.c
index 40a774e..b5fe2ac 100644
--- a/src/listener.c
+++ b/src/listener.c
@@ -433,6 +433,9 @@
* used as a protocol's generic enable_all() primitive, for use after the
* fork(). It puts the listeners into LI_READY or LI_FULL states depending on
* their number of connections. It always returns ERR_NONE.
+ *
+ * Must be called with proto_lock held.
+ *
*/
int enable_all_listeners(struct protocol *proto)
{
@@ -447,6 +450,9 @@
* the polling lists when they are in the LI_READY or LI_FULL states. It is
* intended to be used as a protocol's generic disable_all() primitive. It puts
* the listeners into LI_LISTEN, and always returns ERR_NONE.
+ *
+ * Must be called with proto_lock held.
+ *
*/
int disable_all_listeners(struct protocol *proto)
{
@@ -516,6 +522,9 @@
/* This function closes all listening sockets bound to the protocol <proto>,
* and the listeners end in LI_ASSIGNED state if they were higher. It does not
* detach them from the protocol. It always returns ERR_NONE.
+ *
+ * Must be called with proto_lock held.
+ *
*/
int unbind_all_listeners(struct protocol *proto)
{
@@ -580,14 +589,19 @@
* number of listeners is updated, as well as the global number of listeners
* and jobs. Note that the listener must have previously been unbound. This
* is the generic function to use to remove a listener.
+ *
+ * Will grab the proto_lock.
+ *
*/
void delete_listener(struct listener *listener)
{
HA_SPIN_LOCK(LISTENER_LOCK, &listener->lock);
if (listener->state == LI_ASSIGNED) {
listener->state = LI_INIT;
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
LIST_DEL(&listener->proto_list);
listener->proto->nb_listeners--;
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
_HA_ATOMIC_SUB(&jobs, 1);
_HA_ATOMIC_SUB(&listeners, 1);
}
diff --git a/src/proto_sockpair.c b/src/proto_sockpair.c
index a4faa37..e7dd670 100644
--- a/src/proto_sockpair.c
+++ b/src/proto_sockpair.c
@@ -80,6 +80,9 @@
/* Add <listener> to the list of sockpair listeners (port is ignored). The
* listener's state is automatically updated from LI_INIT to LI_ASSIGNED.
* The number of listeners for the protocol is updated.
+ *
+ * Must be called with proto_lock held.
+ *
*/
static void sockpair_add_listener(struct listener *listener, int port)
{
@@ -97,6 +100,8 @@
* loose them across the fork(). A call to uxst_enable_listeners() is needed
* to complete initialization.
*
+ * Must be called with proto_lock held.
+ *
* The return value is composed from ERR_NONE, ERR_RETRYABLE and ERR_FATAL.
*/
static int sockpair_bind_listeners(struct protocol *proto, char *errmsg, int errlen)
diff --git a/src/proto_tcp.c b/src/proto_tcp.c
index 64ffb83..bcbe27a 100644
--- a/src/proto_tcp.c
+++ b/src/proto_tcp.c
@@ -1103,6 +1103,9 @@
* The sockets will be registered but not added to any fd_set, in order not to
* loose them across the fork(). A call to enable_all_listeners() is needed
* to complete initialization. The return value is composed from ERR_*.
+ *
+ * Must be called with proto_lock held.
+ *
*/
static int tcp_bind_listeners(struct protocol *proto, char *errmsg, int errlen)
{
@@ -1121,6 +1124,9 @@
/* Add <listener> to the list of tcpv4 listeners, on port <port>. The
* listener's state is automatically updated from LI_INIT to LI_ASSIGNED.
* The number of listeners for the protocol is updated.
+ *
+ * Must be called with proto_lock held.
+ *
*/
static void tcpv4_add_listener(struct listener *listener, int port)
{
@@ -1136,6 +1142,9 @@
/* Add <listener> to the list of tcpv6 listeners, on port <port>. The
* listener's state is automatically updated from LI_INIT to LI_ASSIGNED.
* The number of listeners for the protocol is updated.
+ *
+ * Must be called with proto_lock held.
+ *
*/
static void tcpv6_add_listener(struct listener *listener, int port)
{
diff --git a/src/proto_uxst.c b/src/proto_uxst.c
index 66093af..7263240 100644
--- a/src/proto_uxst.c
+++ b/src/proto_uxst.c
@@ -379,6 +379,9 @@
/* Add <listener> to the list of unix stream listeners (port is ignored). The
* listener's state is automatically updated from LI_INIT to LI_ASSIGNED.
* The number of listeners for the protocol is updated.
+ *
+ * Must be called with proto_lock held.
+ *
*/
static void uxst_add_listener(struct listener *listener, int port)
{
@@ -594,6 +597,8 @@
* loose them across the fork(). A call to uxst_enable_listeners() is needed
* to complete initialization.
*
+ * Must be called with proto_lock held.
+ *
* The return value is composed from ERR_NONE, ERR_RETRYABLE and ERR_FATAL.
*/
static int uxst_bind_listeners(struct protocol *proto, char *errmsg, int errlen)
@@ -613,6 +618,9 @@
/* This function stops all listening UNIX sockets bound to the protocol
* <proto>. It does not detaches them from the protocol.
* It always returns ERR_NONE.
+ *
+ * Must be called with proto_lock held.
+ *
*/
static int uxst_unbind_listeners(struct protocol *proto)
{
diff --git a/src/protocol.c b/src/protocol.c
index 96e01c8..ac45cf2 100644
--- a/src/protocol.c
+++ b/src/protocol.c
@@ -18,18 +18,26 @@
#include <common/mini-clist.h>
#include <common/standard.h>
-#include <types/protocol.h>
+#include <proto/protocol.h>
/* List head of all registered protocols */
static struct list protocols = LIST_HEAD_INIT(protocols);
struct protocol *__protocol_by_family[AF_CUST_MAX] = { };
+/* This is the global spinlock we may need to register/unregister listeners or
+ * protocols. Its main purpose is in fact to serialize the rare stop/deinit()
+ * phases.
+ */
+__decl_spinlock(proto_lock);
+
/* Registers the protocol <proto> */
void protocol_register(struct protocol *proto)
{
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
LIST_ADDQ(&protocols, &proto->list);
if (proto->sock_domain >= 0 && proto->sock_domain < AF_CUST_MAX)
__protocol_by_family[proto->sock_domain] = proto;
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
}
/* Unregisters the protocol <proto>. Note that all listeners must have
@@ -37,8 +45,10 @@
*/
void protocol_unregister(struct protocol *proto)
{
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
LIST_DEL(&proto->list);
LIST_INIT(&proto->list);
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
}
/* binds all listeners of all registered protocols. Returns a composition
@@ -50,6 +60,7 @@
int err;
err = 0;
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
list_for_each_entry(proto, &protocols, list) {
if (proto->bind_all) {
err |= proto->bind_all(proto, errmsg, errlen);
@@ -57,6 +68,7 @@
break;
}
}
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
return err;
}
@@ -71,11 +83,13 @@
int err;
err = 0;
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
list_for_each_entry(proto, &protocols, list) {
if (proto->unbind_all) {
err |= proto->unbind_all(proto);
}
}
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
return err;
}
@@ -89,11 +103,13 @@
int err;
err = 0;
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
list_for_each_entry(proto, &protocols, list) {
if (proto->enable_all) {
err |= proto->enable_all(proto);
}
}
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
return err;
}
@@ -107,11 +123,13 @@
int err;
err = 0;
+ HA_SPIN_LOCK(PROTO_LOCK, &proto_lock);
list_for_each_entry(proto, &protocols, list) {
if (proto->disable_all) {
err |= proto->disable_all(proto);
}
}
+ HA_SPIN_UNLOCK(PROTO_LOCK, &proto_lock);
return err;
}