MEDIUM: http-ana: Use a large buffer if necessary when waiting for body

Thanks to previous patches, it is now possible to allocate a large buffer to
store the message payload in the context of the "wait-for-body" action. To
do so, "use-large-buffer" option must be set.

It means now it is no longer necessary to increase the regular buffer size
to be able to get message payloads of some requests or responses.

diff --git a/doc/configuration.txt b/doc/configuration.txt
index a45a0d02b855e6e113c50e88e985c1ff4c1db809..3cb38a50be97f3f0ff4b4d03afa6a03b2d86838f 100644 (file)
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -16603,7 +16603,7 @@ use-service <service-name>
     http-request use-service prometheus-exporter if { path /metrics }
 
 
-wait-for-body time <time> [ at-least <bytes> ]
+wait-for-body time <time> [ at-least <bytes> ] [use-large-buffer]
   Usable in:  QUIC Ini|    TCP RqCon| RqSes| RqCnt| RsCnt|    HTTP Req| Res| Aft
                     - |          -  |   -  |   -  |   -  |          X |  X |  -
 
@@ -16621,6 +16621,10 @@ wait-for-body time <time> [ at-least <bytes> ]
     happens first, this timeout will not occur even if the full body has
     not yet been received.
 
+  "use-large-buffer" option may be set to allocate a large buffer if regular
+  one is to small to store the message body. To be used, "tune.bufsize.large"
+  global option must be defined.
+
   This action may be used as a replacement for "option http-buffer-request".
 
   Arguments :
@@ -16635,7 +16639,7 @@ wait-for-body time <time> [ at-least <bytes> ]
   Example:
     http-request wait-for-body time 1s at-least 1k if METH_POST
 
-  See also : "option http-buffer-request"
+  See also : "option http-buffer-request" and "tune.bufsize.large"
 
 
 wait-for-handshake

diff --git a/include/haproxy/http_ana.h b/include/haproxy/http_ana.h
index 2cc6516c313991ed5f128cc70598437fbcd8cab0..6215c50cc8ce4efb81b50a7d68cc912dc3596180 100644 (file)
--- a/include/haproxy/http_ana.h
+++ b/include/haproxy/http_ana.h
@@ -49,7 +49,7 @@ int http_req_replace_stline(int action, const char *replace, int len,
 int http_res_set_status(unsigned int status, struct ist reason, struct stream *s);
 void http_check_request_for_cacheability(struct stream *s, struct channel *req);
 void http_check_response_for_cacheability(struct stream *s, struct channel *res);
-enum rule_result http_wait_for_msg_body(struct stream *s, struct channel *chn, unsigned int time, unsigned int bytes);
+enum rule_result http_wait_for_msg_body(struct stream *s, struct channel *chn, unsigned int time, unsigned int bytes, unsigned int large_buffer);
 void http_perform_server_redirect(struct stream *s, struct stconn *sc);
 void http_server_error(struct stream *s, struct stconn *sc, int err, int finst, struct http_reply *msg);
 void http_reply_and_close(struct stream *s, short status, struct http_reply *msg);

diff --git a/src/http_act.c b/src/http_act.c
index bfe8321d82ca21901d40ae7a17527eee5771022a..c99cad684fe1c6c42da992fb5334f5cacafb67a8 100644 (file)
--- a/src/http_act.c
+++ b/src/http_act.c
@@ -2379,8 +2379,9 @@ static enum act_return http_action_wait_for_body(struct act_rule *rule, struct p
        struct channel *chn = ((rule->from == ACT_F_HTTP_REQ) ? &s->req : &s->res);
        unsigned int time = (uintptr_t)rule->arg.act.p[0];
        unsigned int bytes = (uintptr_t)rule->arg.act.p[1];
+       unsigned int large_buffer = (uintptr_t)rule->arg.act.p[2];
 
-       switch (http_wait_for_msg_body(s, chn, time, bytes)) {
+       switch (http_wait_for_msg_body(s, chn, time, bytes, large_buffer)) {
        case HTTP_RULE_RES_CONT:
                return ACT_RET_CONT;
        case HTTP_RULE_RES_YIELD:
@@ -2396,6 +2397,21 @@ static enum act_return http_action_wait_for_body(struct act_rule *rule, struct p
        }
 }
 
+/* Check function for 'wait-for-body' HTTP action. The function returns 1 in
+ * success case, otherwise, it returns 0 and err is filled.
+ */
+static int check_http_wait_for_body(struct act_rule *rule, struct proxy *px, char **err)
+{
+       unsigned int large_buffer = (uintptr_t)rule->arg.act.p[2];
+
+       if (large_buffer == 1 && !global.tune.bufsize_large) {
+               memprintf(err, "unable to use large buffers at %s:%d, 'tune.bufsize.large' global parameter must be set",
+                         rule->conf.file, rule->conf.line);
+               return 0;
+       }
+       return 1;
+}
+
 /* Parse a "wait-for-body" action. It returns ACT_RET_PRS_OK on success,
  * ACT_RET_PRS_ERR on error.
  */
@@ -2403,7 +2419,7 @@ static enum act_parse_ret parse_http_wait_for_body(const char **args, int *orig_
                                                   struct act_rule *rule, char **err)
 {
        int cur_arg;
-       unsigned int time, bytes;
+       unsigned int time, bytes, large_buffer;
        const char *res;
 
        cur_arg = *orig_arg;
@@ -2414,6 +2430,7 @@ static enum act_parse_ret parse_http_wait_for_body(const char **args, int *orig_
 
        time = UINT_MAX; /* To be sure it is set */
        bytes = 0; /* Default value, wait all the body */
+       large_buffer = 0; /* Don't use large buffers by default */
        while (*(args[cur_arg])) {
                if (strcmp(args[cur_arg], "time") == 0) {
                        if (!*args[cur_arg + 1]) {
@@ -2447,6 +2464,8 @@ static enum act_parse_ret parse_http_wait_for_body(const char **args, int *orig_
                        }
                        cur_arg++;
                }
+               else if (strcmp(args[cur_arg], "use-large-buffer") == 0)
+                       large_buffer = 1;
                else
                        break;
                cur_arg++;
@@ -2459,11 +2478,13 @@ static enum act_parse_ret parse_http_wait_for_body(const char **args, int *orig_
 
        rule->arg.act.p[0] = (void *)(uintptr_t)time;
        rule->arg.act.p[1] = (void *)(uintptr_t)bytes;
+       rule->arg.act.p[2] = (void *)(uintptr_t)large_buffer;
 
        *orig_arg = cur_arg;
 
        rule->action = ACT_CUSTOM;
        rule->action_ptr = http_action_wait_for_body;
+       rule->check_ptr = check_http_wait_for_body;
        return ACT_RET_PRS_OK;
 }
 

diff --git a/src/http_ana.c b/src/http_ana.c
index 4f6df2ea4e544d0b4cb3df5977ec42ea3644fa48..8a8f3a36980362253bfa2f10590806d48ccc87b2 100644 (file)
--- a/src/http_ana.c
+++ b/src/http_ana.c
@@ -846,7 +846,7 @@ int http_wait_for_request_body(struct stream *s, struct channel *req, int an_bit
        DBG_TRACE_ENTER(STRM_EV_STRM_ANA|STRM_EV_HTTP_ANA, s, txn, &s->txn->req);
 
 
-       switch (http_wait_for_msg_body(s, req, s->be->timeout.httpreq, 0)) {
+       switch (http_wait_for_msg_body(s, req, s->be->timeout.httpreq, 0, 0)) {
        case HTTP_RULE_RES_CONT:
                s->waiting_entity.type = STRM_ENTITY_NONE;
                s->waiting_entity.ptr = NULL;
@@ -4253,7 +4253,7 @@ static int http_handle_stats(struct stream *s, struct channel *req, struct proxy
  * side). All other errors must be handled by the caller.
  */
 enum rule_result http_wait_for_msg_body(struct stream *s, struct channel *chn,
-                                       unsigned int time, unsigned int bytes)
+                                       unsigned int time, unsigned int bytes, unsigned int large_buffer)
 {
        struct session *sess = s->sess;
        struct http_txn *txn = s->txn;
@@ -4286,11 +4286,8 @@ enum rule_result http_wait_for_msg_body(struct stream *s, struct channel *chn,
        /* Now we're are waiting for the payload. We just need to know if all
         * data have been received or if the buffer is full.
         */
-       if ((htx->flags & HTX_FL_EOM) ||
-           htx_get_tail_type(htx) > HTX_BLK_DATA ||
-           channel_htx_full(chn, htx, global.tune.maxrewrite) ||
-           sc_waiting_room(chn_prod(chn)))
-               goto end;
+       if ((htx->flags & HTX_FL_EOM) || htx_get_tail_type(htx) > HTX_BLK_DATA)
+               goto end; /* all data received */
 
        if (bytes) {
                struct htx_blk *blk;
@@ -4305,6 +4302,27 @@ enum rule_result http_wait_for_msg_body(struct stream *s, struct channel *chn,
                }
        }
 
+       if (channel_htx_full(chn, htx, global.tune.maxrewrite) || sc_waiting_room(chn_prod(chn))) {
+               struct buffer lbuf;
+               char *area;
+
+               if (large_buffer == 0 || c_size(chn) == global.tune.bufsize_large)
+                       goto end; /* don't use large buffer or large buffer is full */
+
+               /* normal buffer is full, allocate a large one
+                */
+               area = pool_alloc(pool_head_large_buffer);
+               if (!area)
+                       goto end; /* Allocation failure: TODO must be improved to use buffer_wait */
+               lbuf = b_make(area, global.tune.bufsize_large, 0, 0);
+               htx_xfer_blks(htx_from_buf(&lbuf), htx, htx_used_space(htx), HTX_BLK_UNUSED);
+               htx_to_buf(htx, &chn->buf);
+               b_free(&chn->buf);
+               offer_buffers(s, 1);
+               chn->buf = lbuf;
+               htx = htxbuf(&chn->buf);
+       }
+
        if ((chn->flags & CF_READ_TIMEOUT) || tick_is_expired(chn->analyse_exp, now_ms)) {
                if (!(chn->flags & CF_ISRESP))
                        goto abort_req;