MAJOR: connection: add two new flags to indicate readiness of control/transport
Currently the control and transport layers of a connection are supposed
to be initialized when their respective pointers are not NULL. This will
not work anymore when we plan to reuse connections, because there is an
asymmetry between the accept() side and the connect() side :
- on accept() side, the fd is set first, then the ctrl layer then the
transport layer ; upon error, they must be undone in the reverse order,
then the FD must be closed. The FD must not be deleted if the control
layer was not yet initialized ;
- on the connect() side, the fd is set last and there is no reliable way
to know if it has been initialized or not. In practice it's initialized
to -1 first but this is hackish and supposes that local FDs only will
be used forever. Also, there are even less solutions for keeping trace
of the transport layer's state.
Also it is possible to support delayed close() when something (eg: logs)
tracks some information requiring the transport and/or control layers,
making it even more difficult to clean them.
So the proposed solution is to add two flags to the connection :
- CO_FL_CTRL_READY is set when the control layer is initialized (fd_insert)
and cleared after it's released (fd_delete).
- CO_FL_XPRT_READY is set when the control layer is initialized (xprt->init)
and cleared after it's released (xprt->close).
The functions have been adapted to rely on this and not on the pointers
anymore. conn_xprt_close() was unused and dangerous : it did not close
the control layer (eg: the socket itself) but still marks the transport
layer as closed, preventing any future call to conn_full_close() from
finishing the job.
The problem comes from conn_full_close() in fact. It needs to close the
xprt and ctrl layers independantly. After that we're still having an issue :
we don't know based on ->ctrl alone whether the fd was registered or not.
For this we use the two new flags CO_FL_XPRT_READY and CO_FL_CTRL_READY. We
now rely on this and not on conn->xprt nor conn->ctrl anymore to decide what
remains to be done on the connection.
In order not to miss some flag assignments, we introduce conn_ctrl_init()
to initialize the control layer, register the fd using fd_insert() and set
the flag, and conn_ctrl_close() which unregisters the fd and removes the
flag, but only if the transport layer was closed.
Similarly, at the transport layer, conn_xprt_init() calls ->init and sets
the flag, while conn_xprt_close() checks the flag, calls ->close and clears
the flag, regardless xprt_ctx or xprt_st. This also ensures that the ->init
and the ->close functions are called only once each and in the correct order.
Note that conn_xprt_close() does nothing if the transport layer is still
tracked.
conn_full_close() now simply calls conn_xprt_close() then conn_full_close()
in turn, which do nothing if CO_FL_XPRT_TRACKED is set.
In order to handle the error path, we also provide conn_force_close() which
ignores CO_FL_XPRT_TRACKED and closes the transport and the control layers
in turns. All relevant instances of fd_delete() have been replaced with
conn_force_close(). Now we always know what state the connection is in and
we can expect to split its initialization.
diff --git a/src/checks.c b/src/checks.c
index c38a1c9..2213782 100644
--- a/src/checks.c
+++ b/src/checks.c
@@ -796,7 +796,7 @@
if (conn->flags & CO_FL_ERROR && ((errno && errno != EAGAIN) || !conn->ctrl))
return 1;
- if (!conn->ctrl)
+ if (!(conn->flags & CO_FL_CTRL_READY) || !conn->ctrl)
return 0;
if (getsockopt(conn->t.sock.fd, SOL_SOCKET, SO_ERROR, &skerr, &lskerr) == 0)
@@ -1430,12 +1430,12 @@
/* We're here because nobody wants to handle the error, so we
* sure want to abort the hard way.
*/
- if (conn->ctrl && !(conn->flags & CO_FL_SOCK_RD_SH)) {
+ if ((conn->flags & CO_FL_CTRL_READY) && !(conn->flags & CO_FL_SOCK_RD_SH)) {
if (conn->flags & CO_FL_WAIT_RD || !conn->ctrl->drain || !conn->ctrl->drain(conn->t.sock.fd))
setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
(struct linger *) &nolinger, sizeof(struct linger));
}
- conn_full_close(conn);
+ conn_force_close(conn);
}
return 0;
}
@@ -1650,12 +1650,12 @@
* as a failed response coupled with "observe layer7" caused the
* server state to be suddenly changed.
*/
- if (conn->ctrl && !(conn->flags & CO_FL_SOCK_RD_SH)) {
+ if ((conn->flags & CO_FL_CTRL_READY) && !(conn->flags & CO_FL_SOCK_RD_SH)) {
if (conn->flags & CO_FL_WAIT_RD || !conn->ctrl->drain || !conn->ctrl->drain(conn->t.sock.fd))
setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
(struct linger *) &nolinger, sizeof(struct linger));
}
- conn_full_close(conn);
+ conn_force_close(conn);
}
if (check->result & SRV_CHK_FAILED) /* a failure or timeout detected */
diff --git a/src/connection.c b/src/connection.c
index 6200283..05cad01 100644
--- a/src/connection.c
+++ b/src/connection.c
@@ -163,6 +163,9 @@
{
unsigned int f = c->flags;
+ if (!(c->flags & CO_FL_CTRL_READY))
+ return;
+
/* update read status if needed */
if (unlikely((f & (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD))) {
fd_poll_recv(c->t.sock.fd);
@@ -203,6 +206,9 @@
{
unsigned int f = c->flags;
+ if (!(c->flags & CO_FL_CTRL_READY))
+ return;
+
/* update read status if needed */
if (unlikely((f & (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD))) {
fd_poll_recv(c->t.sock.fd);
@@ -264,6 +270,9 @@
if (conn->flags & CO_FL_SOCK_RD_SH)
goto fail;
+ if (!(conn->flags & CO_FL_CTRL_READY))
+ goto fail;
+
do {
trash.len = recv(conn->t.sock.fd, trash.str, trash.size, MSG_PEEK);
if (trash.len < 0) {
@@ -547,6 +556,9 @@
if (conn->flags & CO_FL_SOCK_WR_SH)
goto out_error;
+ if (!(conn->flags & CO_FL_CTRL_READY))
+ goto out_error;
+
/* The target server expects a PROXY line to be sent first. Retrieving
* local or remote addresses may fail until the connection is established.
*/
diff --git a/src/dumpstats.c b/src/dumpstats.c
index 0df2882..b3d49bf 100644
--- a/src/dumpstats.c
+++ b/src/dumpstats.c
@@ -3833,7 +3833,7 @@
static inline const char *get_conn_ctrl_name(const struct connection *conn)
{
- if (!conn->ctrl)
+ if (!(conn->flags & CO_FL_CTRL_READY) || !conn->ctrl)
return "NONE";
return conn->ctrl->name;
}
@@ -3842,7 +3842,7 @@
{
static char ptr[17];
- if (!conn->xprt)
+ if (!(conn->flags & CO_FL_XPRT_READY) || !conn->xprt)
return "NONE";
if (conn->xprt == &raw_sock)
diff --git a/src/proto_http.c b/src/proto_http.c
index 61ce7ba..70b3a13 100644
--- a/src/proto_http.c
+++ b/src/proto_http.c
@@ -2488,7 +2488,7 @@
req->flags |= CF_READ_DONTWAIT; /* try to get back here ASAP */
s->rep->flags &= ~CF_EXPECT_MORE; /* speed up sending a previous response */
#ifdef TCP_QUICKACK
- if (s->listener->options & LI_O_NOQUICKACK && req->buf->i && objt_conn(s->req->prod->end)) {
+ if (s->listener->options & LI_O_NOQUICKACK && req->buf->i && objt_conn(s->req->prod->end) && (__objt_conn(s->req->prod->end)->flags & CO_FL_CTRL_READY)) {
/* We need more data, we have to re-enable quick-ack in case we
* previously disabled it, otherwise we might cause the client
* to delay next data.
@@ -3015,13 +3015,13 @@
break;
case HTTP_REQ_ACT_SET_TOS:
- if ((cli_conn = objt_conn(s->req->prod->end)))
+ if ((cli_conn = objt_conn(s->req->prod->end)) && (cli_conn->flags & CO_FL_CTRL_READY))
inet_set_tos(cli_conn->t.sock.fd, cli_conn->addr.from, rule->arg.tos);
break;
case HTTP_REQ_ACT_SET_MARK:
#ifdef SO_MARK
- if ((cli_conn = objt_conn(s->req->prod->end)))
+ if ((cli_conn = objt_conn(s->req->prod->end)) && (cli_conn->flags & CO_FL_CTRL_READY))
setsockopt(cli_conn->t.sock.fd, SOL_SOCKET, SO_MARK, &rule->arg.mark, sizeof(rule->arg.mark));
#endif
break;
@@ -3101,13 +3101,13 @@
break;
case HTTP_RES_ACT_SET_TOS:
- if ((cli_conn = objt_conn(s->req->prod->end)))
+ if ((cli_conn = objt_conn(s->req->prod->end)) && (cli_conn->flags & CO_FL_CTRL_READY))
inet_set_tos(cli_conn->t.sock.fd, cli_conn->addr.from, rule->arg.tos);
break;
case HTTP_RES_ACT_SET_MARK:
#ifdef SO_MARK
- if ((cli_conn = objt_conn(s->req->prod->end)))
+ if ((cli_conn = objt_conn(s->req->prod->end)) && (cli_conn->flags & CO_FL_CTRL_READY))
setsockopt(cli_conn->t.sock.fd, SOL_SOCKET, SO_MARK, &rule->arg.mark, sizeof(rule->arg.mark));
#endif
break;
@@ -3930,7 +3930,7 @@
* the client to delay further data.
*/
if ((s->listener->options & LI_O_NOQUICKACK) &&
- cli_conn &&
+ cli_conn && (cli_conn->flags & CO_FL_CTRL_READY) &&
((msg->flags & HTTP_MSGF_TE_CHNK) ||
(msg->body_len > req->buf->i - txn->req.eoh - 2)))
setsockopt(cli_conn->t.sock.fd, IPPROTO_TCP, TCP_QUICKACK, &one, sizeof(one));
diff --git a/src/proto_tcp.c b/src/proto_tcp.c
index de1e4ae..1c55b6a 100644
--- a/src/proto_tcp.c
+++ b/src/proto_tcp.c
@@ -462,16 +462,14 @@
}
}
- fdtab[fd].owner = conn;
conn->flags = CO_FL_WAIT_L4_CONN; /* connection in progress */
conn->flags |= CO_FL_ADDR_TO_SET;
- fdtab[fd].iocb = conn_fd_handler;
- fd_insert(fd);
+ conn_ctrl_init(conn); /* registers the FD */
conn_sock_want_send(conn); /* for connect status */
if (conn_xprt_init(conn) < 0) {
- fd_delete(fd);
+ conn_force_close(conn);
return SN_ERR_RESOURCE;
}
@@ -575,6 +573,9 @@
if (conn->flags & CO_FL_ERROR)
return 0;
+ if (!(conn->flags & CO_FL_CTRL_READY))
+ return 0;
+
if (!(conn->flags & CO_FL_WAIT_L4_CONN))
return 1; /* strange we were called while ready */
diff --git a/src/raw_sock.c b/src/raw_sock.c
index 43cd70a..4dc1c7a 100644
--- a/src/raw_sock.c
+++ b/src/raw_sock.c
@@ -74,6 +74,10 @@
int ret;
int retval = 0;
+
+ if (!(conn->flags & CO_FL_CTRL_READY))
+ return 0;
+
errno = 0;
/* Under Linux, if FD_POLL_HUP is set, we have reached the end.
@@ -189,6 +193,9 @@
{
int ret, done;
+ if (!(conn->flags & CO_FL_CTRL_READY))
+ return 0;
+
done = 0;
while (pipe->data) {
ret = splice(pipe->cons, NULL, conn->t.sock.fd, NULL, pipe->data,
@@ -234,6 +241,9 @@
int ret, done = 0;
int try = count;
+ if (!(conn->flags & CO_FL_CTRL_READY))
+ return 0;
+
errno = 0;
if (unlikely(!(fdtab[conn->t.sock.fd].ev & FD_POLL_IN))) {
@@ -331,6 +341,9 @@
{
int ret, try, done, send_flag;
+ if (!(conn->flags & CO_FL_CTRL_READY))
+ return 0;
+
done = 0;
/* send the largest possible block. For this we perform only one call
* to send() unless the buffer wraps and we exactly fill the first hunk,
diff --git a/src/session.c b/src/session.c
index e509662..88ed5ab 100644
--- a/src/session.c
+++ b/src/session.c
@@ -206,9 +206,8 @@
conn_assign(cli_conn, &sess_conn_cb, l->proto, l->xprt, s);
/* finish initialization of the accepted file descriptor */
- fd_insert(cfd);
- fdtab[cfd].owner = cli_conn;
- fdtab[cfd].iocb = conn_fd_handler;
+ conn_ctrl_init(cli_conn);
+
conn_data_want_recv(cli_conn);
if (conn_xprt_init(cli_conn) < 0)
goto out_free_task;
@@ -242,6 +241,8 @@
session_store_counters(s);
pool_free2(pool2_connection, s->si[1].conn);
out_fail_conn1:
+ s->si[0].conn->flags &= ~CO_FL_XPRT_TRACKED;
+ conn_xprt_close(s->si[0].conn);
pool_free2(pool2_connection, s->si[0].conn);
out_fail_conn0:
pool_free2(pool2_session, s);
@@ -330,7 +331,7 @@
}
/* kill the connection now */
- conn_full_close(conn);
+ conn_force_close(conn);
s->fe->feconn--;
session_store_counters(s);
@@ -654,10 +655,8 @@
http_end_txn(s);
/* ensure the client-side transport layer is destroyed */
- if (cli_conn) {
- cli_conn->flags &= ~CO_FL_XPRT_TRACKED;
- conn_full_close(cli_conn);
- }
+ if (cli_conn)
+ conn_force_close(cli_conn);
for (i = 0; i < s->store_count; i++) {
if (!s->store[i].ts)
@@ -819,8 +818,7 @@
si->exp = TICK_ETERNITY;
si->state = SI_ST_CER;
- srv_conn->flags &= ~CO_FL_XPRT_TRACKED;
- conn_full_close(srv_conn);
+ conn_force_close(srv_conn);
if (si->err_type)
return 0;
diff --git a/src/ssl_sock.c b/src/ssl_sock.c
index 5b8fdb9..e84e191 100644
--- a/src/ssl_sock.c
+++ b/src/ssl_sock.c
@@ -1097,6 +1097,9 @@
if (conn->xprt_ctx)
return 0;
+ if (!(conn->flags & CO_FL_CTRL_READY))
+ return 0;
+
if (global.maxsslconn && sslconns >= global.maxsslconn) {
conn->err_code = CO_ER_SSL_TOO_MANY;
return -1;
@@ -1166,6 +1169,9 @@
{
int ret;
+ if (!(conn->flags & CO_FL_CTRL_READY))
+ return 0;
+
if (!conn->xprt_ctx)
goto out_error;
@@ -1225,7 +1231,7 @@
* TCP sockets. We first try to drain possibly pending
* data to avoid this as much as possible.
*/
- if (conn->ctrl && conn->ctrl->drain)
+ if ((conn->flags & CO_FL_CTRL_READY) && conn->ctrl && conn->ctrl->drain)
conn->ctrl->drain(conn->t.sock.fd);
if (!conn->err_code)
conn->err_code = CO_ER_SSL_HANDSHAKE;
@@ -1276,7 +1282,7 @@
* TCP sockets. We first try to drain possibly pending
* data to avoid this as much as possible.
*/
- if (conn->ctrl && conn->ctrl->drain)
+ if ((conn->flags & CO_FL_CTRL_READY) && conn->ctrl && conn->ctrl->drain)
conn->ctrl->drain(conn->t.sock.fd);
if (!conn->err_code)
conn->err_code = CO_ER_SSL_HANDSHAKE;
diff --git a/src/stream_interface.c b/src/stream_interface.c
index 6932e15..95f5d44 100644
--- a/src/stream_interface.c
+++ b/src/stream_interface.c
@@ -382,6 +382,9 @@
if (conn->flags & CO_FL_SOCK_WR_SH)
goto out_error;
+ if (!(conn->flags & CO_FL_CTRL_READY))
+ goto out_error;
+
/* If we have a PROXY line to send, we'll use this to validate the
* connection, in which case the connection is validated only once
* we've sent the whole proxy line. Otherwise we use connect().
@@ -781,7 +784,7 @@
/* quick close, the socket is alredy shut anyway */
}
else if (si->flags & SI_FL_NOLINGER) {
- if (conn->ctrl) {
+ if ((conn->flags & CO_FL_CTRL_READY) && conn->ctrl) {
setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
(struct linger *) &nolinger, sizeof(struct linger));
}
@@ -802,7 +805,7 @@
*/
if (!(si->flags & SI_FL_NOHALF) || !(si->ib->flags & (CF_SHUTR|CF_DONT_READ))) {
/* We shutdown transport layer */
- if (conn->ctrl)
+ if ((conn->flags & CO_FL_CTRL_READY) && conn->ctrl)
shutdown(conn->t.sock.fd, SHUT_WR);
if (!(si->ib->flags & (CF_SHUTR|CF_DONT_READ))) {
@@ -900,13 +903,13 @@
/* Before calling the data-level operations, we have to prepare
* the polling flags to ensure we properly detect changes.
*/
- if (conn->ctrl)
+ if ((conn->flags & CO_FL_CTRL_READY) && conn->ctrl)
fd_want_send(conn->t.sock.fd);
conn_refresh_polling_flags(conn);
si_conn_send(conn);
- if (conn->flags & CO_FL_ERROR) {
+ if ((conn->flags & CO_FL_CTRL_READY) && (conn->flags & CO_FL_ERROR)) {
/* Write error on the file descriptor */
fd_stop_both(conn->t.sock.fd);
__conn_data_stop_both(conn);
@@ -1256,8 +1259,9 @@
/* we want to immediately forward this close to the write side */
if (si->flags & SI_FL_NOLINGER) {
si->flags &= ~SI_FL_NOLINGER;
- setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
- (struct linger *) &nolinger, sizeof(struct linger));
+ if (conn->flags & CO_FL_CTRL_READY)
+ setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
+ (struct linger *) &nolinger, sizeof(struct linger));
}
/* force flag on ssl to keep session in cache */
if (conn->xprt->shutw)