int conn_local_send_proxy(struct connection *conn, unsigned int flag);
/* inspects c->flags and returns non-zero if DATA ENA changes from the CURR ENA
- * or if the WAIT flags set new flags that were not in CURR POL. Additionally,
+ * or if the WAIT flags are set with their respective ENA flags. Additionally,
* non-zero is also returned if an error was reported on the connection. This
* function is used quite often and is inlined. In order to proceed optimally
* with very little code and CPU cycles, the bits are arranged so that a change
- * can be detected by a simple left shift, a xor, and a mask. This operation
- * detects when POLL:DATA differs from WAIT:CURR. In order to detect the ERROR
- * flag without additional work, we remove it from the copy of the original
- * flags (unshifted) before doing the XOR. This operation is parallelized with
- * the shift and does not induce additional cycles. This explains why we check
- * the error bit shifted left in the mask. Last, the final operation is an AND
- * which the compiler is able to replace with a TEST in boolean conditions. The
- * result is that all these checks are done in 5-6 cycles only and less than 20
- * bytes.
+ * can be detected by a few left shifts, a xor, and a mask. These operations
+ * detect when W&D are both enabled for either direction, when C&D differ for
+ * either direction and when Error is set. The trick consists in first keeping
+ * only the bits we're interested in, since they don't collide when shifted,
+ * and to perform the AND at the end. In practice, the compiler is able to
+ * replace the last AND with a TEST in boolean conditions. This results in
+ * checks that are done in 4-6 cycles and less than 30 bytes.
*/
static inline unsigned int conn_data_polling_changes(const struct connection *c)
{
- unsigned int f = c->flags << 2;
- return ((c->flags & ~(CO_FL_ERROR << 2)) ^ f) &
- ((CO_FL_ERROR<<2)|CO_FL_WAIT_WR|CO_FL_CURR_WR_ENA|CO_FL_WAIT_RD|CO_FL_CURR_RD_ENA) &
- ~(f & (CO_FL_WAIT_WR|CO_FL_WAIT_RD));
+ unsigned int f = c->flags;
+ f &= CO_FL_DATA_WR_ENA | CO_FL_DATA_RD_ENA | CO_FL_CURR_WR_ENA |
+ CO_FL_CURR_RD_ENA | CO_FL_ERROR | CO_FL_WAIT_WR | CO_FL_WAIT_RD;
+
+ f = (f & (f << 2)) | /* test W & D */
+ ((f ^ (f << 1)) & (CO_FL_CURR_WR_ENA|CO_FL_CURR_RD_ENA)); /* test C ^ D */
+ return f & (CO_FL_WAIT_WR | CO_FL_WAIT_RD | CO_FL_CURR_WR_ENA | CO_FL_CURR_RD_ENA | CO_FL_ERROR);
}
/* inspects c->flags and returns non-zero if SOCK ENA changes from the CURR ENA
- * or if the WAIT flags set new flags that were not in CURR POL. Additionally,
+ * or if the WAIT flags are set with their respective ENA flags. Additionally,
* non-zero is also returned if an error was reported on the connection. This
* function is used quite often and is inlined. In order to proceed optimally
* with very little code and CPU cycles, the bits are arranged so that a change
- * can be detected by a simple left shift, a xor, and a mask. This operation
- * detects when CURR:POLL differs from SOCK:WAIT. In order to detect the ERROR
- * flag without additional work, we remove it from the copy of the original
- * flags (unshifted) before doing the XOR. This operation is parallelized with
- * the shift and does not induce additional cycles. This explains why we check
- * the error bit shifted left in the mask. Last, the final operation is an AND
- * which the compiler is able to replace with a TEST in boolean conditions. The
- * result is that all these checks are done in 5-6 cycles only and less than 20
- * bytes.
+ * can be detected by a few left shifts, a xor, and a mask. These operations
+ * detect when W&S are both enabled for either direction, when C&S differ for
+ * either direction and when Error is set. The trick consists in first keeping
+ * only the bits we're interested in, since they don't collide when shifted,
+ * and to perform the AND at the end. In practice, the compiler is able to
+ * replace the last AND with a TEST in boolean conditions. This results in
+ * checks that are done in 4-6 cycles and less than 30 bytes.
*/
static inline unsigned int conn_sock_polling_changes(const struct connection *c)
{
- unsigned int f = c->flags << 2;
- return ((c->flags & ~(CO_FL_ERROR << 2)) ^ f) &
- ((CO_FL_ERROR<<2)|CO_FL_WAIT_WR|CO_FL_SOCK_WR_ENA|CO_FL_WAIT_RD|CO_FL_SOCK_RD_ENA) &
- ~(f & (CO_FL_WAIT_WR|CO_FL_WAIT_RD));
+ unsigned int f = c->flags;
+ f &= CO_FL_SOCK_WR_ENA | CO_FL_SOCK_RD_ENA | CO_FL_CURR_WR_ENA |
+ CO_FL_CURR_RD_ENA | CO_FL_ERROR | CO_FL_WAIT_WR | CO_FL_WAIT_RD;
+
+ f = (f & (f << 3)) | /* test W & S */
+ ((f ^ (f << 2)) & (CO_FL_CURR_WR_ENA|CO_FL_CURR_RD_ENA)); /* test C ^ S */
+ return f & (CO_FL_WAIT_WR | CO_FL_WAIT_RD | CO_FL_CURR_WR_ENA | CO_FL_CURR_RD_ENA | CO_FL_ERROR);
}
/* Automatically updates polling on connection <c> depending on the DATA flags
* - Stopping an I/O event consists in ANDing with ~1.
* - Polling for an I/O event consists in ORing with ~3.
*
- * The last computed state is remembered in CO_FL_CURR_* so that differential
+ * The last ENA state is remembered in CO_FL_CURR_* so that differential
* changes can be applied. After bits are applied, the POLL status bits are
* cleared so that it is possible to detect when an EAGAIN was encountered. For
* pollers that do not support speculative I/O, POLLED is the same as ENABLED
* and the POL flag can safely be ignored. However it makes a difference for
* the connection handler.
*
- * The ENA flags are per-layer (one pair for SOCK, another one for DATA).
- * The POL flags are only for the socket layer since they indicate that EAGAIN
- * was encountered. Thus, the DATA layer uses its own ENA flag and the socket
- * layer's POL flag.
- *
- * The bits are arranged so that it is possible to detect a change by performing
- * only a left shift followed by a xor and applying a mask to the result. The
- * principle is that depending on what we want to check (data polling changes or
- * sock polling changes), we mask different bits. The bits are arranged this way :
- *
- * S(ock) - W(ait) - C(urr) - P(oll) - D(ata)
- *
- * SOCK changes are reported when (S != C) || (W != P) => (S:W) != (C:P)
- * DATA changes are reported when (D != C) || (W != P) => (W:C) != (P:D)
- * The R and W bits are split apart so that we never shift more than 2 bits at
- * a time, allowing move+shift to be done as a single operation on x86.
+ * The ENA flags are per-layer (one pair for SOCK, another one for DATA). The
+ * POL flags are irrelevant to these layers and only reflect the fact that
+ * EAGAIN was encountered, they're materialised by the CO_FL_WAIT_* connection
+ * flags. POL flags always indicate a polling change because it is assumed that
+ * the poller uses a cache and does not always poll.
*/
/* flags for use in connection->flags */
CO_FL_NONE = 0x00000000, /* Just for initialization purposes */
/* Do not change these values without updating conn_*_poll_changes() ! */
- CO_FL_DATA_RD_ENA = 0x00000001, /* receiving data is allowed */
- CO_FL_CURR_RD_POL = 0x00000002, /* receiving needs to poll first */
+ CO_FL_SOCK_RD_ENA = 0x00000001, /* receiving handshakes is allowed */
+ CO_FL_DATA_RD_ENA = 0x00000002, /* receiving data is allowed */
CO_FL_CURR_RD_ENA = 0x00000004, /* receiving is currently allowed */
CO_FL_WAIT_RD = 0x00000008, /* receiving needs to poll first */
- CO_FL_SOCK_RD_ENA = 0x00000010, /* receiving handshakes is allowed */
+
+ CO_FL_SOCK_WR_ENA = 0x00000010, /* sending handshakes is desired */
CO_FL_DATA_WR_ENA = 0x00000020, /* sending data is desired */
- CO_FL_CURR_WR_POL = 0x00000040, /* sending needs to poll first */
- CO_FL_CURR_WR_ENA = 0x00000080, /* sending is currently desired */
- CO_FL_WAIT_WR = 0x00000100, /* sending needs to poll first */
- CO_FL_SOCK_WR_ENA = 0x00000200, /* sending handshakes is desired */
+ CO_FL_CURR_WR_ENA = 0x00000040, /* sending is currently desired */
+ CO_FL_WAIT_WR = 0x00000080, /* sending needs to poll first */
/* These flags are used by data layers to indicate they had to stop
* sending data because a buffer was empty (WAIT_DATA) or stop receiving
}
/* update read status if needed */
- if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_DATA_RD_ENA)) == CO_FL_CURR_RD_ENA)) {
- f &= ~(CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL);
- fd_stop_recv(c->t.sock.fd);
- }
- else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL)) != (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL) &&
- (f & (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD))) {
- f |= (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL);
+ if (unlikely((f & (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_DATA_RD_ENA|CO_FL_WAIT_RD))) {
fd_poll_recv(c->t.sock.fd);
+ f |= CO_FL_CURR_RD_ENA;
}
else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_DATA_RD_ENA)) == CO_FL_DATA_RD_ENA)) {
- f |= CO_FL_CURR_RD_ENA;
fd_want_recv(c->t.sock.fd);
+ f |= CO_FL_CURR_RD_ENA;
+ }
+ else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_DATA_RD_ENA)) == CO_FL_CURR_RD_ENA)) {
+ fd_stop_recv(c->t.sock.fd);
+ f &= ~CO_FL_CURR_RD_ENA;
}
/* update write status if needed */
- if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_DATA_WR_ENA)) == CO_FL_CURR_WR_ENA)) {
- f &= ~(CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL);
- fd_stop_send(c->t.sock.fd);
- }
- else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL)) != (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL) &&
- (f & (CO_FL_DATA_WR_ENA|CO_FL_WAIT_WR)) == (CO_FL_DATA_WR_ENA|CO_FL_WAIT_WR))) {
- f |= (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL);
+ if (unlikely((f & (CO_FL_DATA_WR_ENA|CO_FL_WAIT_WR)) == (CO_FL_DATA_WR_ENA|CO_FL_WAIT_WR))) {
fd_poll_send(c->t.sock.fd);
+ f |= CO_FL_CURR_WR_ENA;
}
else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_DATA_WR_ENA)) == CO_FL_DATA_WR_ENA)) {
- f |= CO_FL_CURR_WR_ENA;
fd_want_send(c->t.sock.fd);
+ f |= CO_FL_CURR_WR_ENA;
+ }
+ else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_DATA_WR_ENA)) == CO_FL_CURR_WR_ENA)) {
+ fd_stop_send(c->t.sock.fd);
+ f &= ~CO_FL_CURR_WR_ENA;
}
c->flags = f;
}
}
/* update read status if needed */
- if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_SOCK_RD_ENA)) == CO_FL_CURR_RD_ENA)) {
- f &= ~(CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL);
- fd_stop_recv(c->t.sock.fd);
- }
- else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL)) != (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL) &&
- (f & (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD))) {
- f |= (CO_FL_CURR_RD_ENA|CO_FL_CURR_RD_POL);
+ if (unlikely((f & (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD)) == (CO_FL_SOCK_RD_ENA|CO_FL_WAIT_RD))) {
fd_poll_recv(c->t.sock.fd);
+ f |= CO_FL_CURR_RD_ENA;
}
else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_SOCK_RD_ENA)) == CO_FL_SOCK_RD_ENA)) {
- f |= CO_FL_CURR_RD_ENA;
fd_want_recv(c->t.sock.fd);
+ f |= CO_FL_CURR_RD_ENA;
+ }
+ else if (unlikely((f & (CO_FL_CURR_RD_ENA|CO_FL_SOCK_RD_ENA)) == CO_FL_CURR_RD_ENA)) {
+ fd_stop_recv(c->t.sock.fd);
+ f &= ~CO_FL_CURR_RD_ENA;
}
/* update write status if needed */
- if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_SOCK_WR_ENA)) == CO_FL_CURR_WR_ENA)) {
- f &= ~(CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL);
- fd_stop_send(c->t.sock.fd);
- }
- else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL)) != (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL) &&
- (f & (CO_FL_SOCK_WR_ENA|CO_FL_WAIT_WR)) == (CO_FL_SOCK_WR_ENA|CO_FL_WAIT_WR))) {
- f |= (CO_FL_CURR_WR_ENA|CO_FL_CURR_WR_POL);
+ if (unlikely((f & (CO_FL_SOCK_WR_ENA|CO_FL_WAIT_WR)) == (CO_FL_SOCK_WR_ENA|CO_FL_WAIT_WR))) {
fd_poll_send(c->t.sock.fd);
+ f |= CO_FL_CURR_WR_ENA;
}
else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_SOCK_WR_ENA)) == CO_FL_SOCK_WR_ENA)) {
- f |= CO_FL_CURR_WR_ENA;
fd_want_send(c->t.sock.fd);
+ f |= CO_FL_CURR_WR_ENA;
+ }
+ else if (unlikely((f & (CO_FL_CURR_WR_ENA|CO_FL_SOCK_WR_ENA)) == CO_FL_CURR_WR_ENA)) {
+ fd_stop_send(c->t.sock.fd);
+ f &= ~CO_FL_CURR_WR_ENA;
}
c->flags = f;
}
projects / thirdparty / haproxy.git / commitdiff
