MEDIUM: mux-quic: implement http-request timeout
Implement http-request timeout for QUIC MUX. It is used when the
connection is opened and is triggered if no HTTP request is received in
time. By HTTP request we mean at least a QUIC stream with a full header
section. Then qcs instance is attached to a sedesc and upper layer is
then responsible to wait for the rest of the request.
This timeout is also used when new QUIC streams are opened during the
connection lifetime to wait for full HTTP request on them. As it's
possible to demux multiple streams in parallel with QUIC, each waiting
stream is registered in a list <opening_list> stored in qcc with <start>
as timestamp in qcs for the stream opening. Once a qcs is attached to a
sedesc, it is removed from <opening_list>. When refreshing MUX timeout,
if <opening_list> is not empty, the first waiting stream is used to set
MUX timeout.
This is efficient as streams are stored in the list in their creation
order so CPU usage is minimal. Also, the size of the list is
automatically restricted by flow control limitation so it should not
grow too much.
Streams are insert in <opening_list> by application protocol layer. This
is because only application protocol can differentiate streams for HTTP
messaging from internal usage. A function qcs_wait_http_req() has been
added to register a request stream by app layer. QUIC MUX can then
remove it from the list in qc_attach_sc().
As a side-note, it was necessary to implement attach qcc_app_ops
callback on hq-interop module to be able to insert a stream in waiting
list. Without this, a BUG_ON statement would be triggered when trying to
remove the stream on sedesc attach. This is to ensure that every
requests streams are registered for http-request timeout.
MUX timeout is explicitely refreshed on MAX_STREAM_DATA and STOP_SENDING
frame parsing to schedule http-request timeout if a new stream has been
instantiated. It was already done on STREAM parsing due to a previous
patch.
diff --git a/include/haproxy/mux_quic-t.h b/include/haproxy/mux_quic-t.h
index 38d2af4..c555ace 100644
--- a/include/haproxy/mux_quic-t.h
+++ b/include/haproxy/mux_quic-t.h
@@ -96,6 +96,7 @@
/* haproxy timeout management */
struct task *task;
+ struct list opening_list; /* list of not already attached streams (http-request timeout) */
int timeout;
int idle_start; /* base time for http-keep-alive timeout */
@@ -165,11 +166,14 @@
struct qc_stream_desc *stream;
struct list el; /* element of qcc.send_retry_list */
+ struct list el_opening; /* element of qcc.opening_list */
struct wait_event wait_event;
struct wait_event *subs;
uint64_t err; /* error code to transmit via RESET_STREAM */
+
+ int start; /* base timestamp for http-request timeout */
};
/* QUIC application layer operations */
diff --git a/include/haproxy/mux_quic.h b/include/haproxy/mux_quic.h
index bf0a18e..d18f1a0 100644
--- a/include/haproxy/mux_quic.h
+++ b/include/haproxy/mux_quic.h
@@ -115,9 +115,36 @@
sess->t_handshake = 0;
sess->t_idle = 0;
+ /* A stream must have been registered for HTTP wait before attaching
+ * it to sedesc. See <qcs_wait_http_req> for more info.
+ */
+ BUG_ON_HOT(!LIST_INLIST(&qcs->el_opening));
+ LIST_DELETE(&qcs->el_opening);
+
return qcs->sd->sc;
}
+/* Register <qcs> stream for http-request timeout. If the stream is not yet
+ * attached in the configured delay, qcc timeout task will be triggered. This
+ * means the full header section was not received in time.
+ *
+ * This function should be called by the application protocol layer on request
+ * streams initialization.
+ */
+static inline void qcs_wait_http_req(struct qcs *qcs)
+{
+ struct qcc *qcc = qcs->qcc;
+
+ /* A stream cannot be registered several times. */
+ BUG_ON_HOT(tick_isset(qcs->start));
+ qcs->start = now_ms;
+
+ /* qcc.opening_list size is limited by flow-control so no custom
+ * restriction is needed here.
+ */
+ LIST_APPEND(&qcc->opening_list, &qcs->el_opening);
+}
+
#endif /* USE_QUIC */
#endif /* _HAPROXY_MUX_QUIC_H */
diff --git a/src/h3.c b/src/h3.c
index f00281e..688f903 100644
--- a/src/h3.c
+++ b/src/h3.c
@@ -1080,6 +1080,7 @@
if (quic_stream_is_bidi(qcs->id)) {
h3s->type = H3S_T_REQ;
h3s->st_req = H3S_ST_REQ_BEFORE;
+ qcs_wait_http_req(qcs);
}
else {
/* stream type must be decoded for unidirectional streams */
diff --git a/src/hq_interop.c b/src/hq_interop.c
index f2933d6..be0287f 100644
--- a/src/hq_interop.c
+++ b/src/hq_interop.c
@@ -164,7 +164,14 @@
return total;
}
+static int hq_interop_attach(struct qcs *qcs, void *conn_ctx)
+{
+ qcs_wait_http_req(qcs);
+ return 0;
+}
+
const struct qcc_app_ops hq_interop_ops = {
.decode_qcs = hq_interop_decode_qcs,
.snd_buf = hq_interop_snd_buf,
+ .attach = hq_interop_attach,
};
diff --git a/src/mux_quic.c b/src/mux_quic.c
index df6e304..e0d0746 100644
--- a/src/mux_quic.c
+++ b/src/mux_quic.c
@@ -130,6 +130,12 @@
qcs->st = QC_SS_IDLE;
qcs->ctx = NULL;
+ /* App callback attach may register the stream for http-request wait.
+ * These fields must be initialed before.
+ */
+ LIST_INIT(&qcs->el_opening);
+ qcs->start = TICK_ETERNITY;
+
/* Allocate transport layer stream descriptor. Only needed for TX. */
if (!quic_stream_is_uni(id) || !quic_stream_is_remote(qcc, id)) {
struct quic_conn *qc = qcc->conn->handle.qc;
@@ -302,18 +308,14 @@
* it with global close_spread delay applied.
*/
- /* TODO implement specific timeouts
- * - http-requset for waiting on incomplete streams
- * - client-fin for graceful shutdown
- */
+ /* TODO implement client/server-fin timeout for graceful shutdown */
/* Frontend timeout management
* - detached streams with data left to send -> default timeout
+ * - stream waiting on incomplete request or no stream yet activated -> timeout http-request
* - idle after stream processing -> timeout http-keep-alive
*/
if (!conn_is_back(qcc->conn)) {
- int timeout;
-
if (qcc->nb_hreq) {
TRACE_DEVEL("one or more requests still in progress", QMUX_EV_QCC_WAKE, qcc->conn);
qcc->task->expire = tick_add_ifset(now_ms, qcc->timeout);
@@ -321,12 +323,29 @@
goto leave;
}
- /* Use http-request timeout if keep-alive timeout not set */
- timeout = tick_isset(px->timeout.httpka) ?
- px->timeout.httpka : px->timeout.httpreq;
+ if (!LIST_ISEMPTY(&qcc->opening_list) || unlikely(!qcc->largest_bidi_r)) {
+ int timeout = px->timeout.httpreq;
+ struct qcs *qcs = NULL;
+ int base_time;
- TRACE_DEVEL("at least one request achieved but none currently in progress", QMUX_EV_QCC_WAKE, qcc->conn);
- qcc->task->expire = tick_add_ifset(qcc->idle_start, timeout);
+ /* Use start time of first stream waiting on HTTP or
+ * qcc idle if no stream not yet used.
+ */
+ if (likely(!LIST_ISEMPTY(&qcc->opening_list)))
+ qcs = LIST_ELEM(qcc->opening_list.n, struct qcs *, el_opening);
+ base_time = qcs ? qcs->start : qcc->idle_start;
+
+ TRACE_DEVEL("waiting on http request", QMUX_EV_QCC_WAKE, qcc->conn, qcs);
+ qcc->task->expire = tick_add_ifset(base_time, timeout);
+ }
+ else {
+ /* Use http-request timeout if keep-alive timeout not set */
+ int timeout = tick_isset(px->timeout.httpka) ?
+ px->timeout.httpka : px->timeout.httpreq;
+
+ TRACE_DEVEL("at least one request achieved but none currently in progress", QMUX_EV_QCC_WAKE, qcc->conn);
+ qcc->task->expire = tick_add_ifset(qcc->idle_start, timeout);
+ }
}
/* fallback to default timeout if frontend specific undefined or for
@@ -1015,6 +1034,9 @@
}
}
+ if (qcc_may_expire(qcc) && !qcc->nb_hreq)
+ qcc_refresh_timeout(qcc);
+
TRACE_LEAVE(QMUX_EV_QCC_RECV, qcc->conn);
return 0;
}
@@ -1064,6 +1086,9 @@
TRACE_DEVEL("receiving STOP_SENDING on stream", QMUX_EV_QCC_RECV|QMUX_EV_QCS_RECV, qcc->conn, qcs);
qcc_reset_stream(qcs, err);
+ if (qcc_may_expire(qcc) && !qcc->nb_hreq)
+ qcc_refresh_timeout(qcc);
+
out:
TRACE_LEAVE(QMUX_EV_QCC_RECV, qcc->conn);
return 0;
@@ -1931,6 +1956,7 @@
qcc->task->expire = tick_add(now_ms, qcc->timeout);
}
qcc_reset_idle_start(qcc);
+ LIST_INIT(&qcc->opening_list);
if (!conn_is_back(conn)) {
if (!LIST_INLIST(&conn->stopping_list)) {