MAJOR: quic: support thread balancing on accept
Before this patch, QUIC protocol used a custom add_listener callback.
This was because a quic_conn instance was allocated before accept. Its
thread affinity was fixed and could not be changed after. The thread was
derived itself from the CID selected by the client which prevent an even
repartition of QUIC connections on multiple threads.
A series of patches was introduced with a lot of changes. The most
important ones :
* removal of affinity between an encoded CID and a thread
* possibility to rebind a quic_conn on a new thread
Thanks to this, it's possible to suppress the custom add_listener
callback. Accept is conducted for QUIC protocol as with the others. A
less loaded thread is selected on listener_accept() and the connection
stack is bind on it. This operation implies that quic_conn instance is
moved to the new thread using the set_affinity QUIC protocol callback.
To reactivate quic_conn instance after thread rebind,
qc_finalize_affinity_rebind() is called after accept on the new thread
by qc_xprt_start() through accept_queue_process() / session_accept_fd().
This should be backported up to 2.7 after a period of observation.
diff --git a/src/proto_quic.c b/src/proto_quic.c
index 0961fc2..8c26377 100644
--- a/src/proto_quic.c
+++ b/src/proto_quic.c
@@ -62,11 +62,11 @@
DECLARE_STATIC_POOL(pool_head_quic_rxbuf, "quic_rxbuf", QUIC_RX_BUFSZ);
-static void quic_add_listener(struct protocol *proto, struct listener *listener);
static int quic_bind_listener(struct listener *listener, char *errmsg, int errlen);
static int quic_connect_server(struct connection *conn, int flags);
static void quic_enable_listener(struct listener *listener);
static void quic_disable_listener(struct listener *listener);
+static int quic_set_affinity(struct connection *conn, int new_tid);
/* Note: must not be declared <const> as its list will be overwritten */
struct protocol proto_quic4 = {
@@ -77,7 +77,7 @@
.listen = quic_bind_listener,
.enable = quic_enable_listener,
.disable = quic_disable_listener,
- .add = quic_add_listener,
+ .add = default_add_listener,
.unbind = default_unbind_listener,
.suspend = default_suspend_listener,
.resume = default_resume_listener,
@@ -85,6 +85,7 @@
.get_src = quic_sock_get_src,
.get_dst = quic_sock_get_dst,
.connect = quic_connect_server,
+ .set_affinity = quic_set_affinity,
/* binding layer */
.rx_suspend = udp_suspend_receiver,
@@ -117,7 +118,7 @@
.listen = quic_bind_listener,
.enable = quic_enable_listener,
.disable = quic_disable_listener,
- .add = quic_add_listener,
+ .add = default_add_listener,
.unbind = default_unbind_listener,
.suspend = default_suspend_listener,
.resume = default_resume_listener,
@@ -125,6 +126,7 @@
.get_src = quic_sock_get_src,
.get_dst = quic_sock_get_dst,
.connect = quic_connect_server,
+ .set_affinity = quic_set_affinity,
/* binding layer */
.rx_suspend = udp_suspend_receiver,
@@ -525,17 +527,6 @@
return SF_ERR_NONE; /* connection is OK */
}
-/* Add listener <listener> to protocol <proto>. Technically speaking we just
- * initialize a few entries which should be doable during quic_bind_listener().
- * The end of the initialization goes on with the default function.
- */
-static void quic_add_listener(struct protocol *proto, struct listener *listener)
-{
- listener->rx.flags |= RX_F_LOCAL_ACCEPT;
-
- default_add_listener(proto, listener);
-}
-
/* Allocate the RX buffers for <l> listener.
* Return 1 if succeeded, 0 if not.
*/
@@ -716,6 +707,12 @@
fd_stop_recv(l->rx.fd);
}
+static int quic_set_affinity(struct connection *conn, int new_tid)
+{
+ struct quic_conn *qc = conn->handle.qc;
+ return qc_set_tid_affinity(qc, new_tid);
+}
+
static int quic_alloc_dghdlrs(void)
{
int i;
diff --git a/src/xprt_quic.c b/src/xprt_quic.c
index 16432db..be9b09c 100644
--- a/src/xprt_quic.c
+++ b/src/xprt_quic.c
@@ -132,6 +132,8 @@
qc = conn->handle.qc;
TRACE_ENTER(QUIC_EV_CONN_NEW, qc);
+ qc_finalize_affinity_rebind(qc);
+
/* mux-quic can now be considered ready. */
qc->mux_state = QC_MUX_READY;