MINOR: compression: Introduce minimum size

This is the introduction of "minsize-req" and "minsize-res".
These two options allow you to set the minimum payload size required for
compression to be applied.
This helps save CPU on both server and client sides when the payload does
not need to be compressed.

diff --git a/doc/configuration.txt b/doc/configuration.txt
index 7f2aa0fbde398d49078b41c02b399163bc096ff6..b7d97d0ecc0ae50ea1fbe4bb76b8117dac8bc1f6 100644 (file)
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -6782,6 +6782,8 @@ compression type <mime type> ...
     * The response contains a "Content-Encoding" header, indicating that the
       response is already compressed (see compression offload)
     * The response contains an invalid "ETag" header or multiple ETag headers
+    * The payload size is smaller than the minimum size
+      (see compression minsize-res)
 
   Note: The compression does not emit the Warning header.
 
@@ -6789,7 +6791,22 @@ compression type <mime type> ...
         compression algo gzip
         compression type text/html text/plain
 
-  See also : "compression offload", "compression direction"
+  See also : "compression offload", "compression direction",
+             "compression minsize-req" and "compression minsize-res"
+
+compression minsize-req <size>
+compression minsize-res <size>
+  Sets the minimum payload size in bytes for compression to be applied.
+
+  May be used in the following contexts: http
+
+  May be used in sections :   defaults | frontend | listen | backend
+                                 yes   |    yes   |   yes  |   yes
+
+  Payloads smaller than this size will not be compressed, avoiding unnecessary
+  CPU overhead for data that would not significantly benefit from compression.
+  "minsize-req" applies on requests and "minsize-res" on responses.
+  The default value is 0.
 
 compression offload
   Makes HAProxy work as a compression offloader only.

diff --git a/include/haproxy/compression-t.h b/include/haproxy/compression-t.h
index b8f118bfe5eef3295741c64a5005f523c0a049f3..12bf12ce5e86cbdd643cd9cad87ec0bd5e30b31b 100644 (file)
--- a/include/haproxy/compression-t.h
+++ b/include/haproxy/compression-t.h
@@ -50,6 +50,8 @@ struct comp {
        struct comp_algo *algo_req;  /* Algo to use for request */
        struct comp_type *types_req; /* Types to be compressed for requests */
        struct comp_type *types_res; /* Types to be compressed for responses */
+       unsigned int minsize_res;    /* Min response body size to be compressed */
+       unsigned int minsize_req;    /* Min request body size to be compressed */
        unsigned int flags;
 };
 

diff --git a/reg-tests/compression/min_size.vtc b/reg-tests/compression/min_size.vtc
new file mode 100644 (file)
index 0000000..52d0716
--- /dev/null
+++ b/reg-tests/compression/min_size.vtc
@@ -0,0 +1,135 @@
+varnishtest "Compression ignores small payloads"
+
+#REQUIRE_OPTION=ZLIB|SLZ
+
+feature ignore_unknown_macro
+
+server s1 {
+        rxreq
+        expect req.url == "/response-lower"
+        expect req.http.accept-encoding == "gzip"
+        txresp \
+          -hdr "Content-Type: text/plain" \
+          -hdr "ETag: \"123\"" \
+          -bodylen 50
+
+        rxreq
+        expect req.url == "/response-equal"
+        expect req.http.accept-encoding == "gzip"
+        txresp \
+          -hdr "Content-Type: text/plain" \
+          -hdr "ETag: \"123\"" \
+          -bodylen 1024
+
+        rxreq
+        expect req.url == "/response-greater"
+        expect req.http.accept-encoding == "gzip"
+        txresp \
+          -hdr "Content-Type: text/plain" \
+          -hdr "ETag: \"123\"" \
+          -bodylen 2000
+
+        rxreq
+        expect req.url == "/request-lower"
+        expect req.http.content-encoding == "<undef>"
+        expect req.method == "POST"
+        expect resp.bodylen == 50
+        txresp
+
+        rxreq
+        expect req.url == "/request-equal"
+        expect req.http.content-encoding == "gzip"
+        expect req.method == "POST"
+        gunzip
+        expect resp.bodylen == 800
+        txresp
+
+        rxreq
+        expect req.url == "/request-greater"
+        expect req.http.content-encoding == "gzip"
+        expect req.method == "POST"
+        gunzip
+        expect resp.bodylen == 2000
+        txresp
+} -start
+
+
+haproxy h1 -conf {
+    global
+        # WT: limit false-positives causing "HTTP header incomplete" due to
+        # idle server connections being randomly used and randomly expiring
+        # under us.
+        tune.idle-pool.shared off
+
+    defaults
+        mode http
+        timeout connect "${HAPROXY_TEST_TIMEOUT-5s}"
+        timeout client  "${HAPROXY_TEST_TIMEOUT-5s}"
+        timeout server  "${HAPROXY_TEST_TIMEOUT-5s}"
+
+    frontend fe-gzip
+        bind "fd@${fe_gzip}"
+        default_backend be-gzip
+
+    backend be-gzip
+        compression direction both
+
+        compression algo-res gzip
+        compression type-res text/plain
+        compression minsize-res 1k
+
+        compression algo-req gzip
+        compression type-req text/plain
+        compression minsize-req 800
+
+        server www ${s1_addr}:${s1_port}
+} -start
+
+client c1 -connect ${h1_fe_gzip_sock} {
+        txreq -url "/response-lower" \
+          -hdr "Accept-Encoding: gzip"
+        rxresp
+        expect resp.status == 200
+        expect resp.http.content-encoding == "<undef>"
+        expect resp.http.etag == "\"123\""
+        expect resp.bodylen == 50
+
+        txreq -url "/response-equal" \
+          -hdr "Accept-Encoding: gzip"
+        rxresp
+        expect resp.status == 200
+        expect resp.http.content-encoding == "gzip"
+        expect resp.http.etag == "W/\"123\""
+        gunzip
+        expect resp.bodylen == 1024
+
+        txreq -url "/response-greater" \
+          -hdr "Accept-Encoding: gzip"
+        rxresp
+        expect resp.status == 200
+        expect resp.http.content-encoding == "gzip"
+        expect resp.http.etag == "W/\"123\""
+        gunzip
+        expect resp.bodylen == 2000
+
+        txreq -method POST \
+          -url "/request-lower" \
+          -hdr "Content-Type: text/plain" \
+          -bodylen 50
+        rxresp
+        expect resp.status == 200
+
+        txreq -method POST \
+          -url "/request-equal" \
+          -hdr "Content-Type: text/plain" \
+          -bodylen 800
+        rxresp
+        expect resp.status == 200
+
+        txreq -method POST \
+          -url "/request-greater" \
+          -hdr "Content-Type: text/plain" \
+          -bodylen 2000
+        rxresp
+        expect resp.status == 200
+} -run

diff --git a/src/flt_http_comp.c b/src/flt_http_comp.c
index e601ff6721557d8684fc379ff047811deaeaa4f3..43e32028422a78f0afa34211888f3b1f7bbbf308 100644 (file)
--- a/src/flt_http_comp.c
+++ b/src/flt_http_comp.c
@@ -137,6 +137,9 @@ comp_prepare_compress_request(struct comp_state *st, struct stream *s, struct ht
        struct http_txn *txn = s->txn;
        struct http_hdr_ctx ctx;
        struct comp_type *comp_type;
+       unsigned int comp_minsize = 0;
+       int32_t pos;
+       unsigned long long len = 0;
 
        ctx.blk = NULL;
        /* Already compressed, don't bother */
@@ -147,6 +150,25 @@ comp_prepare_compress_request(struct comp_state *st, struct stream *s, struct ht
                return;
        comp_type = NULL;
 
+       /* compress only if body size is >= than the min size */
+       if ((s->be->comp && (comp_minsize = s->be->comp->minsize_req)) ||
+               (strm_fe(s)->comp && (comp_minsize = strm_fe(s)->comp->minsize_req))) {
+               for (pos = htx_get_first(htx); pos != -1; pos = htx_get_next(htx, pos)) {
+                       struct htx_blk *blk = htx_get_blk(htx, pos);
+                       enum htx_blk_type type = htx_get_blk_type(blk);
+
+                       if (type == HTX_BLK_TLR || type == HTX_BLK_EOT)
+                               break;
+                       if (type == HTX_BLK_DATA)
+                               len += htx_get_blksz(blk);
+               }
+               if (htx->extra != HTX_UNKOWN_PAYLOAD_LENGTH)
+                       len += htx->extra;
+               /* small requests should not be compressed */
+               if (len < comp_minsize)
+                       goto fail;
+       }
+
        /*
         * We don't want to compress content-types not listed in the "compression type" directive if any. If no content-type was found but configuration
         * requires one, we don't compress either. Backend has the priority.
@@ -624,6 +646,9 @@ select_compression_response_header(struct comp_state *st, struct stream *s, stru
        struct http_txn *txn = s->txn;
        struct http_hdr_ctx ctx;
        struct comp_type *comp_type;
+       unsigned int comp_minsize = 0;
+       int32_t pos;
+       unsigned long long len = 0;
 
        /* no common compression algorithm was found in request header */
        if (st->comp_algo[COMP_DIR_RES] == NULL)
@@ -650,6 +675,25 @@ select_compression_response_header(struct comp_state *st, struct stream *s, stru
        if (!(msg->flags & HTTP_MSGF_XFER_LEN) || msg->flags & HTTP_MSGF_BODYLESS)
                goto fail;
 
+       /* compress only if body size is >= than the min size */
+       if ((s->be->comp && (comp_minsize = s->be->comp->minsize_res)) ||
+               (strm_fe(s)->comp && (comp_minsize = strm_fe(s)->comp->minsize_res))) {
+               for (pos = htx_get_first(htx); pos != -1; pos = htx_get_next(htx, pos)) {
+                       struct htx_blk *blk = htx_get_blk(htx, pos);
+                       enum htx_blk_type type = htx_get_blk_type(blk);
+
+                       if (type == HTX_BLK_TLR || type == HTX_BLK_EOT)
+                               break;
+                       if (type == HTX_BLK_DATA)
+                               len += htx_get_blksz(blk);
+               }
+               if (htx->extra != HTX_UNKOWN_PAYLOAD_LENGTH)
+                       len += htx->extra;
+               /* small responses should not be compressed */
+               if (len < comp_minsize)
+                       goto fail;
+       }
+
        /* content is already compressed */
        ctx.blk = NULL;
        if (http_find_header(htx, ist("Content-Encoding"), &ctx, 1))
@@ -781,6 +825,7 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
 {
        struct comp    *comp;
        int ret = 0;
+       const char *res;
 
        if (proxy->comp == NULL) {
                comp = calloc(1, sizeof(*comp));
@@ -894,6 +939,32 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
                        continue;
                }
        }
+       else if (strcmp(args[1], "minsize-req") == 0) {
+               if (*(args[2]) == 0) {
+                       memprintf(err, "'%s' expects an integer argument.", args[1]);
+                       ret = -1;
+                       goto end;
+               }
+               res = parse_size_err(args[2], &comp->minsize_req);
+               if (res != NULL) {
+                       memprintf(err, "unexpected '%s' after size passed to '%s'", res, args[1]);
+                       ret = -1;
+                       goto end;
+               }
+       }
+       else if (strcmp(args[1], "minsize-res") == 0) {
+               if (*(args[2]) == 0) {
+                       memprintf(err, "'%s' expects an integer argument.", args[1]);
+                       ret = -1;
+                       goto end;
+               }
+               res = parse_size_err(args[2], &comp->minsize_res);
+               if (res != NULL) {
+                       memprintf(err, "unexpected '%s' after size passed to '%s'", res, args[1]);
+                       ret = -1;
+                       goto end;
+               }
+       }
        else if (strcmp(args[1], "direction") == 0) {
                if (!args[2]) {
                        memprintf(err, "'%s' expects 'request', 'response', or 'both'.", args[0]);
@@ -915,7 +986,7 @@ parse_compression_options(char **args, int section, struct proxy *proxy,
                }
        }
        else {
-               memprintf(err, "'%s' expects 'algo', 'type' 'direction' or 'offload'",
+               memprintf(err, "'%s' expects 'algo', 'type', 'direction', 'offload', 'minsize-req' or 'minsize-res'.",
                          args[0]);
                ret = -1;
                goto end;

diff --git a/src/proxy.c b/src/proxy.c
index a1143fc686315fe2739576d68fd1439a6c48e3a4..e0dad6b5189527f9f3992e2f03dfe4b7d327febb 100644 (file)
--- a/src/proxy.c
+++ b/src/proxy.c
@@ -1868,6 +1868,8 @@ static int proxy_defproxy_cpy(struct proxy *curproxy, const struct proxy *defpro
                curproxy->comp->algo_req = defproxy->comp->algo_req;
                curproxy->comp->types_res = defproxy->comp->types_res;
                curproxy->comp->types_req = defproxy->comp->types_req;
+               curproxy->comp->minsize_res = defproxy->comp->minsize_res;
+               curproxy->comp->minsize_req = defproxy->comp->minsize_req;
                curproxy->comp->flags = defproxy->comp->flags;
        }