MINOR: server: move actconns to the per-thread structure
The actconns list creates massive contention on low server counts because
it's in fact a list of streams using a server, all threads compete on the
list's head and it's still possible to see some watchdog panics on 48
threads under extreme contention with 47 threads trying to add and one
thread trying to delete.
Moving this list per thread is trivial because it's only used by
srv_shutdown_streams(), which simply required to iterate over the list.
The field was renamed to "streams" as it's really a list of streams
rather than a list of connections.
diff --git a/include/haproxy/server-t.h b/include/haproxy/server-t.h
index 37a3bce..ad3a40f 100644
--- a/include/haproxy/server-t.h
+++ b/include/haproxy/server-t.h
@@ -205,6 +205,7 @@
/* Each server will have one occurrence of this structure per thread */
struct srv_per_thread {
+ struct mt_list streams; /* streams using this server (used by "shutdown server sessions") */
struct eb_root idle_conns; /* Shareable idle connections */
struct eb_root safe_conns; /* Safe idle connections */
struct eb_root avail_conns; /* Connections in use, but with still new streams available */
@@ -236,8 +237,7 @@
struct be_counters counters; /* statistics counters */
struct eb_root pendconns; /* pending connections */
- struct mt_list actconns; /* active connections (used by "shutdown server sessions") */
- struct srv_per_thread *per_thr; /* array of per-thread stuff such as connections lists, may be null */
+ struct srv_per_thread *per_thr; /* array of per-thread stuff such as connections lists */
unsigned int pool_purge_delay; /* Delay before starting to purge the idle conns pool */
unsigned int low_idle_conns; /* min idle connection count to start picking from other threads */
unsigned int max_idle_conns; /* Max number of connection allowed in the orphan connections list */
diff --git a/include/haproxy/stream.h b/include/haproxy/stream.h
index 948ca73..aa6486e 100644
--- a/include/haproxy/stream.h
+++ b/include/haproxy/stream.h
@@ -292,7 +292,7 @@
* from a conflict with an adjacent MT_LIST_DEL, and using it improves
* the performance by about 3% on 32-cores.
*/
- MT_LIST_ADD(&srv->actconns, &sess->by_srv);
+ MT_LIST_ADD(&srv->per_thr[tid].streams, &sess->by_srv);
HA_ATOMIC_STORE(&sess->srv_conn, srv);
}
diff --git a/src/cfgparse.c b/src/cfgparse.c
index eafd8ca..85995a5 100644
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -3259,6 +3259,7 @@
newsrv->per_thr[i].idle_conns = EB_ROOT;
newsrv->per_thr[i].safe_conns = EB_ROOT;
newsrv->per_thr[i].avail_conns = EB_ROOT;
+ MT_LIST_INIT(&newsrv->per_thr[i].streams);
}
if (newsrv->max_idle_conns != 0) {
diff --git a/src/hlua.c b/src/hlua.c
index 5e5414c..12a4872 100644
--- a/src/hlua.c
+++ b/src/hlua.c
@@ -9175,7 +9175,6 @@
socket_tcp.next = NULL;
socket_tcp.proxy = &socket_proxy;
socket_tcp.obj_type = OBJ_TYPE_SERVER;
- MT_LIST_INIT(&socket_tcp.actconns);
socket_tcp.pendconns = EB_ROOT;
LIST_ADD(&servers_list, &socket_tcp.global_list);
socket_tcp.next_state = SRV_ST_RUNNING; /* early server setup */
@@ -9221,7 +9220,6 @@
socket_ssl.next = NULL;
socket_ssl.proxy = &socket_proxy;
socket_ssl.obj_type = OBJ_TYPE_SERVER;
- MT_LIST_INIT(&socket_ssl.actconns);
socket_ssl.pendconns = EB_ROOT;
LIST_ADD(&servers_list, &socket_ssl.global_list);
socket_ssl.next_state = SRV_ST_RUNNING; /* early server setup */
diff --git a/src/server.c b/src/server.c
index 5844c63..04d2347 100644
--- a/src/server.c
+++ b/src/server.c
@@ -891,10 +891,12 @@
{
struct stream *stream;
struct mt_list *elt1, elt2;
+ int thr;
- mt_list_for_each_entry_safe(stream, &srv->actconns, by_srv, elt1, elt2)
- if (stream->srv_conn == srv)
- stream_shutdown(stream, why);
+ for (thr = 0; thr < global.nbthread; thr++)
+ mt_list_for_each_entry_safe(stream, &srv->per_thr[thr].streams, by_srv, elt1, elt2)
+ if (stream->srv_conn == srv)
+ stream_shutdown(stream, why);
}
/* Shutdown all connections of all backup servers of a proxy. The caller must
@@ -1750,7 +1752,6 @@
srv->obj_type = OBJ_TYPE_SERVER;
srv->proxy = proxy;
- MT_LIST_INIT(&srv->actconns);
srv->pendconns = EB_ROOT;
LIST_ADDQ(&servers_list, &srv->global_list);