MEDIUM: log/balance: support for the "hash" lb algorithm

hash lb algorithm can be configured with the "log-balance hash <cnv_list>"
directive. With this algorithm, the user specifies a converter list with
<cnv_list>.

The produced log message will be passed as-is to the provided converter
list, and the resulting hash will be used to select the log server that
will receive the log message.

diff --git a/doc/configuration.txt b/doc/configuration.txt
index 73652aa9bc11a8942c66d3660ac1caf128ba722c..6f85c1013457cf39213c13fd3f71cb41738a4f3d 100644 (file)
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -8849,6 +8849,19 @@ log-balance <algorithm> [ <arguments> ]
                   pool of available servers as it may avoid the hammering
                   effect that could result from roundrobin in this situation.
 
+      hash        <arguments> should be found in the form: <cnv_list>
+                  e.g.: log-balance hash <cnv_list>
+
+                  Each log message will be passed to the converter list
+                  specified in <cnv_list> (ie: "cnv1,cnv2..."), and it will
+                  then be passed to haproxy hashing function according to
+                  "hash-type" settings. The resulting hash will be used to
+                  select the destination server among the ones declared in the
+                  log backend. The goal of this algorithm is to be able to
+                  extract a key within the final log message using string
+                  converters and then be able to stick to the same server thanks
+                  to the hash. Only "map-based" hashes are supported for now.
+
     <arguments> is an optional list of arguments which may be needed by some
                 algorithms.
 
@@ -8862,6 +8875,7 @@ log-balance <algorithm> [ <arguments> ]
 
         global
           log backend@mylog-rrb local0 # send all logs to mylog-rrb backend
+          log backend@mylog-hash local0 # send all logs to mylog-hash backend
 
         backend mylog-rrb
           mode log
@@ -8870,6 +8884,18 @@ log-balance <algorithm> [ <arguments> ]
           server s1 udp@127.0.0.1:514 # will receive 50% of log messages
           server s2 udp@127.0.0.1:514
 
+        backend mylog-hash
+          mode log
+
+          # extract "METHOD URL PROTO" at the end of the log message,
+          # and let haproxy hash it so that log messages generated from
+          # similar requests get sent to the same syslog server:
+          log-balance hash 'field(-2,\")'
+
+          # server list here
+          server s1 127.0.0.1:514
+          #...
+
 log-format <string>
   Specifies the log format string to use for traffic logs
   May be used in sections:    defaults | frontend | listen | backend

diff --git a/include/haproxy/backend.h b/include/haproxy/backend.h
index a5623494fd2d1fe4bf5485a8a1131dd49be4b2a6..581c5a07dfb91bacf619629ee26af21babf70da3 100644 (file)
--- a/include/haproxy/backend.h
+++ b/include/haproxy/backend.h
@@ -147,6 +147,8 @@ static inline int srv_lb_status_changed(const struct server *srv)
  */
 void set_backend_down(struct proxy *be);
 
+unsigned int gen_hash(const struct proxy* px, const char* key, unsigned long len);
+
 #endif /* _HAPROXY_BACKEND_H */
 
 /*

diff --git a/src/backend.c b/src/backend.c
index 809d922271008a7d640c8f9aa7f8c32444e82929..94e0032fc9f324b05ffe23dc2d412d4a3cc67b23 100644 (file)
--- a/src/backend.c
+++ b/src/backend.c
@@ -70,7 +70,7 @@ int be_lastsession(const struct proxy *be)
 }
 
 /* helper function to invoke the correct hash method */
-static unsigned int gen_hash(const struct proxy* px, const char* key, unsigned long len)
+unsigned int gen_hash(const struct proxy* px, const char* key, unsigned long len)
 {
        unsigned int hash;
 
@@ -2855,8 +2855,19 @@ int backend_parse_log_balance(const char **args, char **err, struct proxy *curpr
                curproxy->lbprm.algo &= ~BE_LB_ALGO;
                curproxy->lbprm.algo |= BE_LB_ALGO_RND;
        }
+       else if (strcmp(args[0], "hash") == 0) {
+               if (!*args[1]) {
+                       memprintf(err, "%s requires a converter list.", args[0]);
+                       return -1;
+               }
+               curproxy->lbprm.algo &= ~BE_LB_ALGO;
+               curproxy->lbprm.algo |= BE_LB_ALGO_SMP;
+
+               ha_free(&curproxy->lbprm.arg_str);
+               curproxy->lbprm.arg_str = strdup(args[1]);
+       }
        else {
-               memprintf(err, "only supports 'roundrobin', 'sticky', 'random', options");
+               memprintf(err, "only supports 'roundrobin', 'sticky', 'random', 'hash' options");
                return -1;
        }
        return 0;

diff --git a/src/cfgparse-listen.c b/src/cfgparse-listen.c
index f039a4eb37c485689862ee2e9797b31263d8aed4..7ed67aa32854698499567b6c2d440a6aa91233f6 100644 (file)
--- a/src/cfgparse-listen.c
+++ b/src/cfgparse-listen.c
@@ -2576,8 +2576,8 @@ stats_error_parsing:
                 */
                curproxy->lbprm.algo &= ~(BE_LB_HASH_TYPE | BE_LB_HASH_FUNC | BE_LB_HASH_MOD);
 
-               if (curproxy->mode != PR_MODE_TCP && curproxy->mode != PR_MODE_HTTP) {
-                       ha_alert("parsing [%s:%d] : '%s' requires TCP or HTTP mode.\n", file, linenum, args[0]);
+               if (curproxy->mode != PR_MODE_TCP && curproxy->mode != PR_MODE_HTTP && curproxy->mode != PR_MODE_SYSLOG) {
+                       ha_alert("parsing [%s:%d] : '%s' requires TCP, HTTP or LOG mode.\n", file, linenum, args[0]);
                        err_code |= ERR_ALERT | ERR_FATAL;
                        goto out;
                }

diff --git a/src/cfgparse.c b/src/cfgparse.c
index 963894e0c4780774d3e10de05b7fb40dfa0dd3e5..ef18bfbeefb20451ed848bdc333f2cd60ca18d71 100644 (file)
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -3510,8 +3510,11 @@ out_uri_auth_compat:
                        curproxy->conf.args.line = 0;
                }
 
-               /* "balance hash" needs to compile its expression */
-               if ((curproxy->lbprm.algo & BE_LB_ALGO) == BE_LB_ALGO_SMP) {
+               /* "balance hash" needs to compile its expression
+                * (log backends will handle this in proxy log postcheck)
+                */
+               if (curproxy->mode != PR_MODE_SYSLOG &&
+                   (curproxy->lbprm.algo & BE_LB_ALGO) == BE_LB_ALGO_SMP) {
                        int idx = 0;
                        const char *args[] = {
                                curproxy->lbprm.arg_str,

diff --git a/src/log.c b/src/log.c
index 53011f7f7c958d0be24a38edb2a9ff1328ee650e..bcae3a6973e5b0acdcd5754514ad3443b72f70ad 100644 (file)
--- a/src/log.c
+++ b/src/log.c
@@ -42,6 +42,7 @@
 #include <haproxy/stconn.h>
 #include <haproxy/stream.h>
 #include <haproxy/time.h>
+#include <haproxy/hash.h>
 #include <haproxy/tools.h>
 
 /* global recv logs counter */
@@ -907,6 +908,65 @@ static int postcheck_log_backend(struct proxy *be)
        be->srv_act = 0;
        be->srv_bck = 0;
 
+       /* "log-balance hash" needs to compile its expression */
+       if ((be->lbprm.algo & BE_LB_ALGO) == BE_LB_ALGO_SMP) {
+               struct sample_expr *expr;
+               char *expr_str = NULL;
+               char *err_str = NULL;
+               int idx = 0;
+
+               /* only map-based hash method is supported for now */
+               if ((be->lbprm.algo & BE_LB_HASH_TYPE) != BE_LB_HASH_MAP) {
+                       memprintf(&msg, "unsupported hash method (from \"hash-type\")");
+                       err_code |= ERR_ALERT | ERR_FATAL;
+                       goto end;
+               }
+
+               /* a little bit of explanation about what we're going to do here:
+                * as the user gave us a list of converters, instead of the fetch+conv list
+                * tuple as we're used to, we need to insert a dummy fetch at the start of
+                * the converter list so that sample_parse_expr() is able to properly parse
+                * the expr. We're explicitly using str() as dummy fetch, since the input
+                * sample that will be passed to the converter list at runtime will be a
+                * string (the log message about to be sent). Doing so allows sample_parse_expr()
+                * to ensure that the provided converters will be compatible with string type.
+                */
+               memprintf(&expr_str, "str(dummy),%s", be->lbprm.arg_str);
+               if (!expr_str) {
+                       memprintf(&msg, "memory error during converter list argument parsing (from \"log-balance hash\")");
+                       err_code |= ERR_ALERT | ERR_FATAL;
+                       goto end;
+               }
+               expr = sample_parse_expr((char*[]){expr_str, NULL}, &idx,
+                                        be->conf.file,
+                                        be->conf.line,
+                                        &err_str, NULL, NULL);
+               if (!expr) {
+                       memprintf(&msg, "%s (from converter list argument in \"log-balance hash\")", err_str);
+                       ha_free(&err_str);
+                       err_code |= ERR_ALERT | ERR_FATAL;
+                       ha_free(&expr_str);
+                       goto end;
+               }
+
+               /* We expect the log_message->conv_list expr to resolve as a binary-compatible
+                * value because its output will be passed to gen_hash() to compute the hash.
+                *
+                * So we check the last converter's output type to ensure that it can be
+                * converted into the expected type. Invalid output type will result in an
+                * error to prevent unexpected results during runtime.
+                */
+               if (sample_casts[smp_expr_output_type(expr)][SMP_T_BIN] == NULL) {
+                       memprintf(&msg, "invalid output type at the end of converter list for \"log-balance hash\" directive");
+                       err_code |= ERR_ALERT | ERR_FATAL;
+                       release_sample_expr(expr);
+                       ha_free(&expr_str);
+                       goto end;
+               }
+               ha_free(&expr_str);
+               be->lbprm.expr = expr;
+       }
+
        /* finish the initialization of proxy's servers */
        srv = be->srv;
        while (srv) {
@@ -2120,6 +2180,24 @@ static inline void __do_send_log_backend(struct proxy *be, struct log_header hdr
                /* random mode */
                targetid = statistical_prng() % nb_srv;
        }
+       else if ((be->lbprm.algo & BE_LB_ALGO) == BE_LB_ALGO_SMP) {
+               struct sample result;
+
+               /* log-balance hash */
+               memset(&result, 0, sizeof(result));
+               result.data.type = SMP_T_STR;
+               result.flags = SMP_F_CONST;
+               result.data.u.str.area = message;
+               result.data.u.str.data = size;
+               result.data.u.str.size = size + 1; /* with terminating NULL byte */
+               if (sample_process_cnv(be->lbprm.expr, &result)) {
+                       /* gen_hash takes binary input, ensure that we provide such value to it */
+                       if (result.data.type == SMP_T_BIN || sample_casts[result.data.type][SMP_T_BIN]) {
+                               sample_casts[result.data.type][SMP_T_BIN](&result);
+                               targetid = gen_hash(be, result.data.u.str.area, result.data.u.str.data) % nb_srv;
+                       }
+               }
+       }
 
  skip_lb: