OPTIM: channel: inline channel_forward's fast path

Most calls to channel_forward() are performed with short byte counts and
are already optimized in channel_forward() taking just a few instructions.
Thus it's a waste of CPU cycles to call a function for this, let's just
inline the short byte count case and fall back to the common one for
remaining situations.

Doing so has increased the chunked encoding parser's performance by 12% !

diff --git a/include/proto/channel.h b/include/proto/channel.h
index 80bd39d1df8918f04cebb462c5bfb3f39f6cca03..2364214a1661b8be006f295fff210c4eebc68628 100644 (file)
--- a/include/proto/channel.h
+++ b/include/proto/channel.h
@@ -39,7 +39,7 @@ extern struct pool_head *pool2_channel;
 /* perform minimal intializations, report 0 in case of error, 1 if OK. */
 int init_channel();
 
-unsigned long long channel_forward(struct channel *chn, unsigned long long bytes);
+unsigned long long __channel_forward(struct channel *chn, unsigned long long bytes);
 
 /* SI-to-channel functions working with buffers */
 int bi_putblk(struct channel *chn, const char *str, int len);
@@ -62,6 +62,32 @@ static inline void channel_init(struct channel *chn)
        chn->flags = 0;
 }
 
+/* Schedule up to <bytes> more bytes to be forwarded via the channel without
+ * notifying the owner task. Any data pending in the buffer are scheduled to be
+ * sent as well, in the limit of the number of bytes to forward. This must be
+ * the only method to use to schedule bytes to be forwarded. If the requested
+ * number is too large, it is automatically adjusted. The number of bytes taken
+ * into account is returned. Directly touching ->to_forward will cause lockups
+ * when buf->o goes down to zero if nobody is ready to push the remaining data.
+ */
+static inline unsigned long long channel_forward(struct channel *chn, unsigned long long bytes)
+{
+       /* hint: avoid comparisons on long long for the fast case, since if the
+        * length does not fit in an unsigned it, it will never be forwarded at
+        * once anyway.
+        */
+       if (bytes <= ~0U) {
+               unsigned int bytes32 = bytes;
+
+               if (bytes32 <= chn->buf->i) {
+                       /* OK this amount of bytes might be forwarded at once */
+                       b_adv(chn->buf, bytes32);
+                       return bytes;
+               }
+       }
+       return __channel_forward(chn, bytes);
+}
+
 /*********************************************************************/
 /* These functions are used to compute various channel content sizes */
 /*********************************************************************/

diff --git a/src/channel.c b/src/channel.c
index 6e945d4c4d0a841c6f90661e04fee22a7b63355e..3e2a3f35f32314ef5029caa679b64f01af20ef14 100644 (file)
--- a/src/channel.c
+++ b/src/channel.c
@@ -39,27 +39,10 @@ int init_channel()
  * into account is returned. Directly touching ->to_forward will cause lockups
  * when buf->o goes down to zero if nobody is ready to push the remaining data.
  */
-unsigned long long channel_forward(struct channel *chn, unsigned long long bytes)
+unsigned long long __channel_forward(struct channel *chn, unsigned long long bytes)
 {
        unsigned int new_forward;
        unsigned int forwarded;
-       unsigned int bytes32;
-
-       bytes32 = bytes;
-
-       /* hint: avoid comparisons on long long for the fast case, since if the
-        * length does not fit in an unsigned it, it will never be forwarded at
-        * once anyway.
-        */
-       if (bytes <= ~0U) {
-               if (bytes32 <= chn->buf->i) {
-                       /* OK this amount of bytes might be forwarded at once */
-                       if (!bytes32)
-                               return 0;
-                       b_adv(chn->buf, bytes32);
-                       return bytes;
-               }
-       }
 
        forwarded = chn->buf->i;
        b_adv(chn->buf, chn->buf->i);