MEDIUM: cli: Add support for dynamically allocated payloads

It is now possible to deal with too big payload to fit in a buffer, without
changing the buffer size. By default, a payload up to 128 KB can be
dynamically allocated. "tune.cli.max-payload-size" global parameter can be
used to change this value, with some caution for huge values.

For CLI command handler functions, there is no change at all. A pointer on
the payload is still passed as parameter. Internally, an area is allocated
for the payload only if it is too big.

The payload pattern used to detect the end of the payload is part from the
allocated area.

diff --git a/doc/configuration.txt b/doc/configuration.txt
index f6f0c4a2c6eefcb65b8c0be0e86e44cc0859dd3e..9d3099570ef18f6c84b9930d35413b3ced4d8151 100644 (file)
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -1884,6 +1884,7 @@ The following keywords are supported in the "global" section :
    - tune.bufsize
    - tune.bufsize.large
    - tune.bufsize.small
+   - tune.cli.max-payload-size
    - tune.comp.maxlevel
    - tune.defaults.purge
    - tune.disable-fast-forward
@@ -4192,6 +4193,22 @@ tune.bufsize.small <size>
 
   See also: option use-small-buffers
 
+tune.cli.max-payload-size <size>
+  Sets the maximum size allowed for the payload passed to a command on the CLI.
+
+  On the CLI, a command line is limited by the buffer size. It means all
+  commands and their arguments must fit in a buffer to be processed, excluding
+  the payload that can be passed to the last command of the command line. This
+  payload can be allocated into a dedicated area if necessary. Its size is
+  limited by this parameter. The default value is 128KB.
+
+  While it should be high enough for most usage, if this value is changed, it
+  must be carefully chosen. A huge value can have impact on the HAProxy
+  performance. Depending on the command, a huge payload can be quite long to
+  process and can possibly trigger the watchdog.
+
+  Please consult the management manual for details about the CLI.
+
 tune.comp.maxlevel <number>
   Sets the maximum compression level. The compression level affects CPU
   usage during compression. This value affects CPU usage during compression.

diff --git a/doc/management.txt b/doc/management.txt
index e1ae99ca62236d6707ac40d1ca57981c36ac818e..7d4d5e46309d89948efd4203e064083582664288 100644 (file)
--- a/doc/management.txt
+++ b/doc/management.txt
@@ -1647,15 +1647,18 @@ a payload, it needs to end with an empty line.
 The payload pattern can be customized in order to change the way the payload
 ends. In order to end a payload with something else than an empty line, a
 customized pattern can be set between '<<' and '\n'. Only 7 characters can be
-used in addiction to '<<', otherwise this won't be considered a payload.
+used in addition to '<<', otherwise this won't be considered a payload.
 For example, to use a PEM file that contains empty lines and comments:
 
   # echo -e "set ssl cert common.pem <<%EOF%\n$(cat common.pem)\n%EOF%\n" | \
   socat /var/run/haproxy.stat -
 
-Limitations do exist: the length of the whole buffer passed to the CLI must
-not be greater than tune.bfsize and the pattern "<<" must not be glued to the
-last word of the line.
+Limitations do exist: The pattern "<<" must not be glued to the last word of the
+line. The length of a command line must not be greater than tune.bufsize,
+including the pattern starting the payload, but excluding the payload
+itself. The payload size is limited to 128KB by default. This can be changed by
+setting "tune.cli.max-payload-size" global parameter, with some cautions. Note
+the pattern marking the end of the payload is part of this limit.
 
 When entering a payload while in interactive mode, the prompt will change from
 "> " to "+ ".

diff --git a/include/haproxy/applet-t.h b/include/haproxy/applet-t.h
index dd8e93eb3fe46ff8e1c518c2ca7290c22c6024ee..57178c5dac4b4fd274d739bc44e62a7d6891556d 100644 (file)
--- a/include/haproxy/applet-t.h
+++ b/include/haproxy/applet-t.h
@@ -125,8 +125,8 @@ struct appctx {
                int severity_output;    /* used within the cli_io_handler to format severity output of informational feedback */
                int level;              /* the level of CLI which can be lowered dynamically */
                char *payload_pat;      /* Pointer on the payload pattern. NULL if no payload */
+               uint32_t max_payload_sz;/* Max size allowed for dynamic payload. 0 if not allowed */
                uint32_t anon_key;      /* the key to anonymise with the hash in cli */
-               /* XXX 4 unused bytes here */
                int (*io_handler)(struct appctx *appctx);  /* used within the cli_io_handler when st0 = CLI_ST_CALLBACK */
                void (*io_release)(struct appctx *appctx); /* used within the cli_io_handler when st0 = CLI_ST_CALLBACK,
                                                              if the command is terminated or the session released */

diff --git a/include/haproxy/cli-t.h b/include/haproxy/cli-t.h
index 76e935dbbd20f5f187652e44c0a030a115eeafc2..14fa8c843002e285636f606a597798b818869755 100644 (file)
--- a/include/haproxy/cli-t.h
+++ b/include/haproxy/cli-t.h
@@ -49,6 +49,7 @@
 #define APPCTX_CLI_ST1_PROMPT  (1 << 4) /* display prompt */
 #define APPCTX_CLI_ST1_TIMED   (1 << 5) /* display timer in prompt */
 #define APPCTX_CLI_ST1_YIELD   (1 << 6) /* forced yield between commands */
+#define APPCTX_CLI_ST1_DYN_PAYLOAD (1 << 7) /* the payload was dynamically allocated  */
 
 #define CLI_PREFIX_KW_NB 5
 #define CLI_MAX_MATCHES 5

diff --git a/include/haproxy/global-t.h b/include/haproxy/global-t.h
index a303954c031c12871eafaaa23f477d4061b0ee13..2ffd8305f70385cbb0131ada765b5d33de3659d6 100644 (file)
--- a/include/haproxy/global-t.h
+++ b/include/haproxy/global-t.h
@@ -215,6 +215,7 @@ struct global {
                int default_shards; /* default shards for listeners, or -1 (by-thread) or -2 (by-group) */
                uint max_checks_per_thread; /* if >0, no more than this concurrent checks per thread */
                uint ring_queues;   /* if >0, #ring queues, otherwise equals #thread groups */
+               uint cli_max_payload_sz; /* The max payload size for the CLI */
                enum threadgroup_takeover tg_takeover; /* Policy for threadgroup takeover */
        } tune;
        struct {

diff --git a/src/cli.c b/src/cli.c
index 687687934627279a29b05916cd188f67393baf49..fa180df504279f009884e654069d0bdfe2abf86d 100644 (file)
--- a/src/cli.c
+++ b/src/cli.c
@@ -653,6 +653,34 @@ static int cli_parse_global(char **args, int section_type, struct proxy *curpx,
        return 0;
 }
 
+/* This function parses "tune.cli.max-payload-sze" statement in the "global"
+ * section. It returns -1 if there is any error, otherwise zero. If it returns
+ * -1, it will write an error message into the <err> buffer which will be
+ * preallocated. The trailing '\n' must not be written. The function must be
+ * called with <args> pointing to the first word after "stats".
+ */
+static int cli_parse_global_max_payload_size(char **args, int section_type, struct proxy *curpx,
+                                            const struct proxy *defpx, const char *file, int line,
+                                            char **err)
+{
+       const char *res;
+
+       if (too_many_args(1, args, err, NULL))
+               return -1;
+
+       if (*(args[1]) == 0) {
+               memprintf(err, "'%s' expects a size argument.", args[0]);
+               return -1;
+       }
+       res = parse_size_err(args[1], &global.tune.cli_max_payload_sz);
+       if (res != NULL) {
+               memprintf(err, "unexpected '%s' after size passed to '%s'", res, args[0]);
+               return -1;
+       }
+       return 0;
+}
+
+
 /*
  * This function exports the bound addresses of a <frontend> in the environment
  * variable <varname>. Those addresses are separated by semicolons and prefixed
@@ -950,6 +978,7 @@ int cli_init(struct appctx *appctx)
        appctx->cli_ctx.payload_pat = NULL;
        appctx->cli_ctx.cmdline = NULL;
        appctx->cli_ctx.payload = BUF_NULL;
+       appctx->cli_ctx.max_payload_sz = global.tune.cli_max_payload_sz;
 
        /* Wakeup the applet ASAP. */
         applet_need_more_data(appctx);
@@ -957,6 +986,29 @@ int cli_init(struct appctx *appctx)
 
 }
 
+int cli_try_realloc_payload(struct appctx *appctx, struct buffer *buf, size_t new_size)
+{
+       char *old, *new;
+
+       if (new_size > appctx->cli_ctx.max_payload_sz)
+               new_size = appctx->cli_ctx.max_payload_sz;
+
+       old = (appctx->st1 & APPCTX_CLI_ST1_DYN_PAYLOAD) ? b_orig(buf) : NULL;
+       new = my_realloc2(old, new_size);
+       if (!new) {
+               *buf = BUF_NULL;
+               appctx->st1 &= ~APPCTX_CLI_ST1_DYN_PAYLOAD;
+               return -1;
+       }
+
+       if (!(appctx->st1 & APPCTX_CLI_ST1_DYN_PAYLOAD)) {
+               memcpy(new, b_orig(buf), b_data(buf));
+               appctx->st1 |= APPCTX_CLI_ST1_DYN_PAYLOAD;
+       }
+       *buf = b_make(new, new_size, 0, b_data(buf));
+       return 0;
+}
+
 int cli_parse_cmdline(struct appctx *appctx)
 {
        char *str;
@@ -971,9 +1023,7 @@ int cli_parse_cmdline(struct appctx *appctx)
                appctx->cli_ctx.cmdline = alloc_trash_chunk();
                if (!appctx->cli_ctx.cmdline) {
                        cli_err(appctx, "Failed to alloc a buffer to process the command line.\n");
-                       applet_set_error(appctx);
-                       b_reset(&appctx->inbuf);
-                       goto end;
+                       goto error;
                }
        }
 
@@ -995,10 +1045,26 @@ int cli_parse_cmdline(struct appctx *appctx)
 
                if (!len) {
                        if (!b_room(buf) || (b_data(&appctx->inbuf) > b_room(buf) - 1)) {
-                               cli_err(appctx, "The command line is too big for the buffer size. Please change tune.bufsize in the configuration to use a bigger command.\n");
-                               applet_set_error(appctx);
-                               b_reset(&appctx->inbuf);
-                               goto end;
+                               if (!(appctx->st1 & APPCTX_CLI_ST1_PAYLOAD)) {
+                                       /* The command line is too big and payload, if any, cannot be dynamic */
+                                       cli_err(appctx, "The command line is too big for the buffer size."
+                                               " Please change 'tune.bufsize' in the configuration to use a bigger command.\n");
+                                       goto error;
+                               }
+
+                               /* try to reallocate a bigger payload */
+                               if (!appctx->cli_ctx.max_payload_sz ||
+                                   b_data(&appctx->inbuf) + b_data(buf) + 1 > appctx->cli_ctx.max_payload_sz ||
+                                   cli_try_realloc_payload(appctx, buf, 2 * (b_data(&appctx->inbuf) + b_data(buf))) == -1) {
+                                       /* Payload is too big or realloc failed */
+                                       cli_err(appctx, "The payload is too big."
+                                               " Please change 'tune.cli.max-payload-size' in the configuration to use a bigger payload.\n");
+                                       goto error;
+                               }
+
+                               /* A bigger payload buffer was allcated, wait for more data */
+                               b_xfer(buf, &appctx->inbuf, b_data(&appctx->inbuf));
+                               applet_fl_clr(appctx, APPCTX_FL_INBLK_FULL);
                        }
                        break;
                }
@@ -1105,6 +1171,11 @@ int cli_parse_cmdline(struct appctx *appctx)
        if (appctx->st0 != CLI_ST_PARSE_CMDLINE)
                ret = 1;
        return ret;
+
+ error:
+       applet_set_error(appctx);
+       b_reset(&appctx->inbuf);
+       goto end;
 }
 
 /* This I/O handler runs as an applet embedded in a stream connector. It is
@@ -1300,10 +1371,12 @@ void cli_io_handler(struct appctx *appctx)
                        if (appctx->st1 & APPCTX_CLI_ST1_LASTCMD) {
                                applet_reset_svcctx(appctx);
                                free_trash_chunk(appctx->cli_ctx.cmdline);
+                               if (appctx->st1 & APPCTX_CLI_ST1_DYN_PAYLOAD)
+                                       free(b_orig(&appctx->cli_ctx.payload));
                                appctx->cli_ctx.payload_pat = NULL;
                                appctx->cli_ctx.cmdline = NULL;
                                appctx->cli_ctx.payload = BUF_NULL;
-                               appctx->st1 &= ~APPCTX_CLI_ST1_LASTCMD;
+                               appctx->st1 &= ~(APPCTX_CLI_ST1_LASTCMD|APPCTX_CLI_ST1_DYN_PAYLOAD);
                                if (appctx->st1 & APPCTX_CLI_ST1_INTER) {
                                        appctx->st0 = CLI_ST_PARSE_CMDLINE;
                                        applet_will_consume(appctx);
@@ -1347,6 +1420,10 @@ void cli_io_handler(struct appctx *appctx)
 static void cli_release_handler(struct appctx *appctx)
 {
        free_trash_chunk(appctx->cli_ctx.cmdline);
+       if (appctx->st1 & APPCTX_CLI_ST1_DYN_PAYLOAD) {
+               free(b_orig(&appctx->cli_ctx.payload));
+               appctx->st1 &= ~APPCTX_CLI_ST1_DYN_PAYLOAD;
+       }
 
        if (appctx->cli_ctx.io_release) {
                EXEC_CTX_NO_RET(appctx->cli_ctx.kw->exec_ctx, appctx->cli_ctx.io_release(appctx));
@@ -4042,6 +4119,7 @@ INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);
 
 static struct cfg_kw_list cfg_kws = {ILH, {
        { CFG_GLOBAL, "stats", cli_parse_global },
+       { CFG_GLOBAL, "tune.cli.max-payload-size", cli_parse_global_max_payload_size },
        { 0, NULL, NULL },
 }};
 

diff --git a/src/haproxy.c b/src/haproxy.c
index 292679dfe2d7b5affd5ec8dd49304accd4630503..32702ea742151a902ceacfa83c52c8e301778cd0 100644 (file)
--- a/src/haproxy.c
+++ b/src/haproxy.c
@@ -201,6 +201,7 @@ struct global global = {
 #endif
                .nb_stk_ctr = MAX_SESS_STKCTR,
                .default_shards = -2, /* by-group */
+               .cli_max_payload_sz = 128 * 1024,
        },
 #ifdef USE_OPENSSL
 #ifdef DEFAULT_MAXSSLCONN