MAJOR: connection: remove the CO_FL_CURR_*_POL flag
This is the first step of a series of changes aiming at making the
polling totally event-driven. This first change consists in only
remembering at the connection level whether an FD was enabled or not,
regardless of the fact it was being polled or cached. From now on, an
EAGAIN will always be considered as a change so that the pollers are
able to manage a cache and to flush it based on such events. One of
the noticeable effect is that conn_fd_handler() is called once more
per session (6 instead of 5 min) but other update functions are less
called.
Note that the performance loss caused by this change at the moment is
quite significant, around 2.5%, but the change is needed to have SSL
working correctly in all situations, even when data were read from the
socket and stored in the invisible cache, waiting for some room in the
channel's buffer.
diff --git a/include/proto/connection.h b/include/proto/connection.h
index 4c9e610..0236d95 100644
--- a/include/proto/connection.h
+++ b/include/proto/connection.h
@@ -96,49 +96,51 @@
int conn_local_send_proxy(struct connection *conn, unsigned int flag);
/* inspects c->flags and returns non-zero if DATA ENA changes from the CURR ENA
- * or if the WAIT flags set new flags that were not in CURR POL. Additionally,
+ * or if the WAIT flags are set with their respective ENA flags. Additionally,
* non-zero is also returned if an error was reported on the connection. This
* function is used quite often and is inlined. In order to proceed optimally
* with very little code and CPU cycles, the bits are arranged so that a change
- * can be detected by a simple left shift, a xor, and a mask. This operation
- * detects when POLL:DATA differs from WAIT:CURR. In order to detect the ERROR
- * flag without additional work, we remove it from the copy of the original
- * flags (unshifted) before doing the XOR. This operation is parallelized with
- * the shift and does not induce additional cycles. This explains why we check
- * the error bit shifted left in the mask. Last, the final operation is an AND
- * which the compiler is able to replace with a TEST in boolean conditions. The
- * result is that all these checks are done in 5-6 cycles only and less than 20
- * bytes.
+ * can be detected by a few left shifts, a xor, and a mask. These operations
+ * detect when W&D are both enabled for either direction, when C&D differ for
+ * either direction and when Error is set. The trick consists in first keeping
+ * only the bits we're interested in, since they don't collide when shifted,
+ * and to perform the AND at the end. In practice, the compiler is able to
+ * replace the last AND with a TEST in boolean conditions. This results in
+ * checks that are done in 4-6 cycles and less than 30 bytes.
*/
static inline unsigned int conn_data_polling_changes(const struct connection *c)
{
- unsigned int f = c->flags << 2;
- return ((c->flags & ~(CO_FL_ERROR << 2)) ^ f) &
- ((CO_FL_ERROR<<2)|CO_FL_WAIT_WR|CO_FL_CURR_WR_ENA|CO_FL_WAIT_RD|CO_FL_CURR_RD_ENA) &
- ~(f & (CO_FL_WAIT_WR|CO_FL_WAIT_RD));
+ unsigned int f = c->flags;
+ f &= CO_FL_DATA_WR_ENA | CO_FL_DATA_RD_ENA | CO_FL_CURR_WR_ENA |
+ CO_FL_CURR_RD_ENA | CO_FL_ERROR | CO_FL_WAIT_WR | CO_FL_WAIT_RD;
+
+ f = (f & (f << 2)) | /* test W & D */
+ ((f ^ (f << 1)) & (CO_FL_CURR_WR_ENA|CO_FL_CURR_RD_ENA)); /* test C ^ D */
+ return f & (CO_FL_WAIT_WR | CO_FL_WAIT_RD | CO_FL_CURR_WR_ENA | CO_FL_CURR_RD_ENA | CO_FL_ERROR);
}
/* inspects c->flags and returns non-zero if SOCK ENA changes from the CURR ENA
- * or if the WAIT flags set new flags that were not in CURR POL. Additionally,
+ * or if the WAIT flags are set with their respective ENA flags. Additionally,
* non-zero is also returned if an error was reported on the connection. This
* function is used quite often and is inlined. In order to proceed optimally
* with very little code and CPU cycles, the bits are arranged so that a change
- * can be detected by a simple left shift, a xor, and a mask. This operation
- * detects when CURR:POLL differs from SOCK:WAIT. In order to detect the ERROR
- * flag without additional work, we remove it from the copy of the original
- * flags (unshifted) before doing the XOR. This operation is parallelized with
- * the shift and does not induce additional cycles. This explains why we check
- * the error bit shifted left in the mask. Last, the final operation is an AND
- * which the compiler is able to replace with a TEST in boolean conditions. The
- * result is that all these checks are done in 5-6 cycles only and less than 20
- * bytes.
+ * can be detected by a few left shifts, a xor, and a mask. These operations
+ * detect when W&S are both enabled for either direction, when C&S differ for
+ * either direction and when Error is set. The trick consists in first keeping
+ * only the bits we're interested in, since they don't collide when shifted,
+ * and to perform the AND at the end. In practice, the compiler is able to
+ * replace the last AND with a TEST in boolean conditions. This results in
+ * checks that are done in 4-6 cycles and less than 30 bytes.
*/
static inline unsigned int conn_sock_polling_changes(const struct connection *c)
{
- unsigned int f = c->flags << 2;
- return ((c->flags & ~(CO_FL_ERROR << 2)) ^ f) &
- ((CO_FL_ERROR<<2)|CO_FL_WAIT_WR|CO_FL_SOCK_WR_ENA|CO_FL_WAIT_RD|CO_FL_SOCK_RD_ENA) &
- ~(f & (CO_FL_WAIT_WR|CO_FL_WAIT_RD));
+ unsigned int f = c->flags;
+ f &= CO_FL_SOCK_WR_ENA | CO_FL_SOCK_RD_ENA | CO_FL_CURR_WR_ENA |
+ CO_FL_CURR_RD_ENA | CO_FL_ERROR | CO_FL_WAIT_WR | CO_FL_WAIT_RD;
+
+ f = (f & (f << 3)) | /* test W & S */
+ ((f ^ (f << 2)) & (CO_FL_CURR_WR_ENA|CO_FL_CURR_RD_ENA)); /* test C ^ S */
+ return f & (CO_FL_WAIT_WR | CO_FL_WAIT_RD | CO_FL_CURR_WR_ENA | CO_FL_CURR_RD_ENA | CO_FL_ERROR);
}
/* Automatically updates polling on connection <c> depending on the DATA flags
diff --git a/include/types/connection.h b/include/types/connection.h
index 299093b..b828ffa 100644
--- a/include/types/connection.h
+++ b/include/types/connection.h
@@ -55,29 +55,18 @@
* - Stopping an I/O event consists in ANDing with ~1.
* - Polling for an I/O event consists in ORing with ~3.
*
- * The last computed state is remembered in CO_FL_CURR_* so that differential
+ * The last ENA state is remembered in CO_FL_CURR_* so that differential
* changes can be applied. After bits are applied, the POLL status bits are
* cleared so that it is possible to detect when an EAGAIN was encountered. For
* pollers that do not support speculative I/O, POLLED is the same as ENABLED
* and the POL flag can safely be ignored. However it makes a difference for
* the connection handler.
*
- * The ENA flags are per-layer (one pair for SOCK, another one for DATA).
- * The POL flags are only for the socket layer since they indicate that EAGAIN
- * was encountered. Thus, the DATA layer uses its own ENA flag and the socket
- * layer's POL flag.
- *
- * The bits are arranged so that it is possible to detect a change by performing
- * only a left shift followed by a xor and applying a mask to the result. The
- * principle is that depending on what we want to check (data polling changes or
- * sock polling changes), we mask different bits. The bits are arranged this way :
- *
- * S(ock) - W(ait) - C(urr) - P(oll) - D(ata)
- *
- * SOCK changes are reported when (S != C) || (W != P) => (S:W) != (C:P)
- * DATA changes are reported when (D != C) || (W != P) => (W:C) != (P:D)
- * The R and W bits are split apart so that we never shift more than 2 bits at
- * a time, allowing move+shift to be done as a single operation on x86.
+ * The ENA flags are per-layer (one pair for SOCK, another one for DATA). The
+ * POL flags are irrelevant to these layers and only reflect the fact that
+ * EAGAIN was encountered, they're materialised by the CO_FL_WAIT_* connection
+ * flags. POL flags always indicate a polling change because it is assumed that
+ * the poller uses a cache and does not always poll.
*/
/* flags for use in connection->flags */
@@ -85,16 +74,15 @@
CO_FL_NONE = 0x00000000, /* Just for initialization purposes */
/* Do not change these values without updating conn_*_poll_changes() ! */
- CO_FL_DATA_RD_ENA = 0x00000001, /* receiving data is allowed */
- CO_FL_CURR_RD_POL = 0x00000002, /* receiving needs to poll first */
+ CO_FL_SOCK_RD_ENA = 0x00000001, /* receiving handshakes is allowed */
+ CO_FL_DATA_RD_ENA = 0x00000002, /* receiving data is allowed */
CO_FL_CURR_RD_ENA = 0x00000004, /* receiving is currently allowed */
CO_FL_WAIT_RD = 0x00000008, /* receiving needs to poll first */
- CO_FL_SOCK_RD_ENA = 0x00000010, /* receiving handshakes is allowed */
+
+ CO_FL_SOCK_WR_ENA = 0x00000010, /* sending handshakes is desired */
CO_FL_DATA_WR_ENA = 0x00000020, /* sending data is desired */
- CO_FL_CURR_WR_POL = 0x00000040, /* sending needs to poll first */
- CO_FL_CURR_WR_ENA = 0x00000080, /* sending is currently desired */
- CO_FL_WAIT_WR = 0x00000100, /* sending needs to poll first */
- CO_FL_SOCK_WR_ENA = 0x00000200, /* sending handshakes is desired */
+ CO_FL_CURR_WR_ENA = 0x00000040, /* sending is currently desired */
+ CO_FL_WAIT_WR = 0x00000080, /* sending needs to poll first */
/* These flags are used by data layers to indicate they had to stop
* sending data because a buffer was empty (WAIT_DATA) or stop receiving
diff --git a/src/connection.c b/src/connection.c
index d698e69..3cdc58f 100644
--- a/src/connection.c
+++ b/src/connection.c
@@ -175,33 +175,31 @@
}
/* update read status if needed */
- if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_DATA_RD_ENA)) == CO_FL_CURR_RD_ENA)) {
- f &= ~(CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL);
- fd_stop_recv(c->t.sock.fd);
- }
- else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL)) != (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL) &&
- (f & (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD))) {
- f |= (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL);
+ if (unlikely((f & (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD))) {
fd_poll_recv(c->t.sock.fd);
+ f |= CO_FL_CURR_RD_ENA;
}
else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_DATA_RD_ENA)) == CO_FL_DATA_RD_ENA)) {
- f |= CO_FL_CURR_RD_ENA;
fd_want_recv(c->t.sock.fd);
+ f |= CO_FL_CURR_RD_ENA;
+ }
+ else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_DATA_RD_ENA)) == CO_FL_CURR_RD_ENA)) {
+ fd_stop_recv(c->t.sock.fd);
+ f &= ~CO_FL_CURR_RD_ENA;
}
/* update write status if needed */
- if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_DATA_WR_ENA)) == CO_FL_CURR_WR_ENA)) {
- f &= ~(CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL);
- fd_stop_send(c->t.sock.fd);
- }
- else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL)) != (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL) &&
- (f & (CO_FL_DATA_WR_ENA|CO_FL_WAIT_WR)) == (CO_FL_DATA_WR_ENA|CO_FL_WAIT_WR))) {
- f |= (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL);
+ if (unlikely((f & (CO_FL_DATA_WR_ENA|CO_FL_WAIT_WR)) == (CO_FL_DATA_WR_ENA|CO_FL_WAIT_WR))) {
fd_poll_send(c->t.sock.fd);
+ f |= CO_FL_CURR_WR_ENA;
}
else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_DATA_WR_ENA)) == CO_FL_DATA_WR_ENA)) {
- f |= CO_FL_CURR_WR_ENA;
fd_want_send(c->t.sock.fd);
+ f |= CO_FL_CURR_WR_ENA;
+ }
+ else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_DATA_WR_ENA)) == CO_FL_CURR_WR_ENA)) {
+ fd_stop_send(c->t.sock.fd);
+ f &= ~CO_FL_CURR_WR_ENA;
}
c->flags = f;
}
@@ -225,33 +223,31 @@
}
/* update read status if needed */
- if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_SOCK_RD_ENA)) == CO_FL_CURR_RD_ENA)) {
- f &= ~(CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL);
- fd_stop_recv(c->t.sock.fd);
- }
- else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL)) != (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL) &&
- (f & (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD))) {
- f |= (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL);
+ if (unlikely((f & (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD))) {
fd_poll_recv(c->t.sock.fd);
+ f |= CO_FL_CURR_RD_ENA;
}
else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_SOCK_RD_ENA)) == CO_FL_SOCK_RD_ENA)) {
- f |= CO_FL_CURR_RD_ENA;
fd_want_recv(c->t.sock.fd);
+ f |= CO_FL_CURR_RD_ENA;
+ }
+ else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_SOCK_RD_ENA)) == CO_FL_CURR_RD_ENA)) {
+ fd_stop_recv(c->t.sock.fd);
+ f &= ~CO_FL_CURR_RD_ENA;
}
/* update write status if needed */
- if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_SOCK_WR_ENA)) == CO_FL_CURR_WR_ENA)) {
- f &= ~(CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL);
- fd_stop_send(c->t.sock.fd);
- }
- else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL)) != (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL) &&
- (f & (CO_FL_SOCK_WR_ENA|CO_FL_WAIT_WR)) == (CO_FL_SOCK_WR_ENA|CO_FL_WAIT_WR))) {
- f |= (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL);
+ if (unlikely((f & (CO_FL_SOCK_WR_ENA|CO_FL_WAIT_WR)) == (CO_FL_SOCK_WR_ENA|CO_FL_WAIT_WR))) {
fd_poll_send(c->t.sock.fd);
+ f |= CO_FL_CURR_WR_ENA;
}
else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_SOCK_WR_ENA)) == CO_FL_SOCK_WR_ENA)) {
- f |= CO_FL_CURR_WR_ENA;
fd_want_send(c->t.sock.fd);
+ f |= CO_FL_CURR_WR_ENA;
+ }
+ else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_SOCK_WR_ENA)) == CO_FL_CURR_WR_ENA)) {
+ fd_stop_send(c->t.sock.fd);
+ f &= ~CO_FL_CURR_WR_ENA;
}
c->flags = f;
}