MAJOR: connection: add two new flags to indicate readiness of control/transport
Currently the control and transport layers of a connection are supposed
to be initialized when their respective pointers are not NULL. This will
not work anymore when we plan to reuse connections, because there is an
asymmetry between the accept() side and the connect() side :
- on accept() side, the fd is set first, then the ctrl layer then the
transport layer ; upon error, they must be undone in the reverse order,
then the FD must be closed. The FD must not be deleted if the control
layer was not yet initialized ;
- on the connect() side, the fd is set last and there is no reliable way
to know if it has been initialized or not. In practice it's initialized
to -1 first but this is hackish and supposes that local FDs only will
be used forever. Also, there are even less solutions for keeping trace
of the transport layer's state.
Also it is possible to support delayed close() when something (eg: logs)
tracks some information requiring the transport and/or control layers,
making it even more difficult to clean them.
So the proposed solution is to add two flags to the connection :
- CO_FL_CTRL_READY is set when the control layer is initialized (fd_insert)
and cleared after it's released (fd_delete).
- CO_FL_XPRT_READY is set when the control layer is initialized (xprt->init)
and cleared after it's released (xprt->close).
The functions have been adapted to rely on this and not on the pointers
anymore. conn_xprt_close() was unused and dangerous : it did not close
the control layer (eg: the socket itself) but still marks the transport
layer as closed, preventing any future call to conn_full_close() from
finishing the job.
The problem comes from conn_full_close() in fact. It needs to close the
xprt and ctrl layers independantly. After that we're still having an issue :
we don't know based on ->ctrl alone whether the fd was registered or not.
For this we use the two new flags CO_FL_XPRT_READY and CO_FL_CTRL_READY. We
now rely on this and not on conn->xprt nor conn->ctrl anymore to decide what
remains to be done on the connection.
In order not to miss some flag assignments, we introduce conn_ctrl_init()
to initialize the control layer, register the fd using fd_insert() and set
the flag, and conn_ctrl_close() which unregisters the fd and removes the
flag, but only if the transport layer was closed.
Similarly, at the transport layer, conn_xprt_init() calls ->init and sets
the flag, while conn_xprt_close() checks the flag, calls ->close and clears
the flag, regardless xprt_ctx or xprt_st. This also ensures that the ->init
and the ->close functions are called only once each and in the correct order.
Note that conn_xprt_close() does nothing if the transport layer is still
tracked.
conn_full_close() now simply calls conn_xprt_close() then conn_full_close()
in turn, which do nothing if CO_FL_XPRT_TRACKED is set.
In order to handle the error path, we also provide conn_force_close() which
ignores CO_FL_XPRT_TRACKED and closes the transport and the control layers
in turns. All relevant instances of fd_delete() have been replaced with
conn_force_close(). Now we always know what state the connection is in and
we can expect to split its initialization.
diff --git a/include/proto/connection.h b/include/proto/connection.h
index 21d57c9..e3b6e89 100644
--- a/include/proto/connection.h
+++ b/include/proto/connection.h
@@ -43,30 +43,64 @@
int conn_recv_proxy(struct connection *conn, int flag);
int make_proxy_line(char *buf, int buf_len, struct sockaddr_storage *src, struct sockaddr_storage *dst);
-/* calls the init() function of the transport layer if any.
+/* Calls the init() function of the transport layer if any and if not done yet,
+ * and sets the CO_FL_XPRT_READY flag to indicate it was properly initialized.
* Returns <0 in case of error.
*/
static inline int conn_xprt_init(struct connection *conn)
{
- if (conn->xprt && conn->xprt->init)
- return conn->xprt->init(conn);
- return 0;
+ int ret = 0;
+
+ if (!(conn->flags & CO_FL_XPRT_READY) && conn->xprt && conn->xprt->init)
+ ret = conn->xprt->init(conn);
+
+ if (ret >= 0)
+ conn->flags |= CO_FL_XPRT_READY;
+
+ return ret;
}
-/* Calls the close() function of the transport layer if any, and always unsets
- * the transport layer. However this is not done if the CO_FL_XPRT_TRACKED flag
- * is set, which allows logs to take data from the transport layer very late if
- * needed.
+/* Calls the close() function of the transport layer if any and if not done
+ * yet, and clears the CO_FL_XPRT_READY flag. However this is not done if the
+ * CO_FL_XPRT_TRACKED flag is set, which allows logs to take data from the
+ * transport layer very late if needed.
*/
static inline void conn_xprt_close(struct connection *conn)
{
- if (conn->xprt && !(conn->flags & CO_FL_XPRT_TRACKED)) {
- if (conn->xprt->close)
+ if ((conn->flags & (CO_FL_XPRT_READY|CO_FL_XPRT_TRACKED)) == CO_FL_XPRT_READY) {
+ if (conn->xprt && conn->xprt->close)
conn->xprt->close(conn);
- conn->xprt = NULL;
+ conn->flags &= ~CO_FL_XPRT_READY;
+ }
+}
+
+/* Initializes the connection's control layer which essentially consists in
+ * registering the file descriptor for polling and setting the CO_FL_CTRL_READY
+ * flag.
+ */
+static inline void conn_ctrl_init(struct connection *conn)
+{
+ if (!(conn->flags & CO_FL_CTRL_READY)) {
+ int fd = conn->t.sock.fd;
+
+ fd_insert(fd);
+ fdtab[fd].owner = conn;
+ fdtab[fd].iocb = conn_fd_handler;
+ conn->flags |= CO_FL_CTRL_READY;
}
}
+/* Deletes the FD if the transport layer is already gone. Once done,
+ * it then removes the CO_FL_CTRL_READY flag.
+ */
+static inline void conn_ctrl_close(struct connection *conn)
+{
+ if ((conn->flags & (CO_FL_XPRT_READY|CO_FL_CTRL_READY)) == CO_FL_CTRL_READY) {
+ fd_delete(conn->t.sock.fd);
+ conn->flags &= ~CO_FL_CTRL_READY;
+ }
+}
+
/* If the connection still has a transport layer, then call its close() function
* if any, and delete the file descriptor if a control layer is set. This is
* used to close everything at once and atomically. However this is not done if
@@ -75,13 +109,22 @@
*/
static inline void conn_full_close(struct connection *conn)
{
- if (conn->xprt && !(conn->flags & CO_FL_XPRT_TRACKED)) {
- if (conn->xprt->close)
- conn->xprt->close(conn);
- if (conn->ctrl)
- fd_delete(conn->t.sock.fd);
- conn->xprt = NULL;
- }
+ conn_xprt_close(conn);
+ conn_ctrl_close(conn);
+}
+
+/* Force to close the connection whatever the tracking state. This is mainly
+ * used on the error path where the tracking does not make sense.
+ */
+static inline void conn_force_close(struct connection *conn)
+{
+ if ((conn->flags & CO_FL_XPRT_READY) && conn->xprt && conn->xprt->close)
+ conn->xprt->close(conn);
+
+ if (conn->flags & CO_FL_CTRL_READY)
+ fd_delete(conn->t.sock.fd);
+
+ conn->flags &= ~(CO_FL_XPRT_READY|CO_FL_CTRL_READY);
}
/* Update polling on connection <c>'s file descriptor depending on its current
@@ -121,7 +164,7 @@
{
conn->flags &= ~(CO_FL_WAIT_ROOM | CO_FL_WAIT_RD | CO_FL_WAIT_DATA | CO_FL_WAIT_WR);
- if (conn->ctrl) {
+ if ((conn->flags & CO_FL_CTRL_READY) && conn->ctrl) {
unsigned int flags = conn->flags & ~(CO_FL_CURR_RD_ENA | CO_FL_CURR_WR_ENA);
if (fd_ev_is_set(conn->t.sock.fd, DIR_RD))
@@ -470,7 +513,7 @@
if (conn->flags & CO_FL_ADDR_FROM_SET)
return;
- if (!conn->ctrl || !conn->ctrl->get_src)
+ if (!(conn->flags & CO_FL_CTRL_READY) || !conn->ctrl || !conn->ctrl->get_src)
return;
if (conn->ctrl->get_src(conn->t.sock.fd, (struct sockaddr *)&conn->addr.from,
@@ -486,7 +529,7 @@
if (conn->flags & CO_FL_ADDR_TO_SET)
return;
- if (!conn->ctrl || !conn->ctrl->get_dst)
+ if (!(conn->flags & CO_FL_CTRL_READY) || !conn->ctrl || !conn->ctrl->get_dst)
return;
if (conn->ctrl->get_dst(conn->t.sock.fd, (struct sockaddr *)&conn->addr.to,