MEDIUM: sample: Enhances converter "bytes" to take variable names as arguments

Prior to this commit, converter "bytes" takes only integer values as
arguments.  After this commit, it can take variable names as inputs.
This allows us to dynamically determine the offset/length and capture
them in variables.  These variables can then be used with the converter.
Example use case: parsing a token present in a request header.

diff --git a/doc/configuration.txt b/doc/configuration.txt
index d49d359a22f0f9e9f9736cd1d714d2ccd31a14b1..37fb004c54ab408b803e1d6a2d588bd1a115ee9a 100644 (file)
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -18059,7 +18059,21 @@ bool
 bytes(<offset>[,<length>])
   Extracts some bytes from an input binary sample. The result is a binary
   sample starting at an offset (in bytes) of the original sample and
-  optionally truncated at the given length.
+  optionally truncated at the given length. <offset> and <length> can be numeric
+  values or variable names. The converter returns an empty sample if either
+  <offset> or <length> is invalid. Invalid <offset> means a negative value or a
+  value >= length of the input sample. Invalid <length> means a negative value
+  or, in some cases, a value bigger than the length of the input sample.
+
+  Example:
+      http-request set-var(txn.input) req.hdr(input) # let's say input is "012345"
+
+      http-response set-header bytes_0 "%[var(txn.input),bytes(0)]"  # outputs "012345"
+      http-response set-header bytes_1_3 "%[var(txn.input),bytes(1,3)]"  # outputs "123"
+
+      http-response set-var(txn.var_start) int(1)
+      http-response set-var(txn.var_length) int(3)
+      http-response set-header bytes_var1_var3    "%[var(txn.input),bytes(txn.var_start,txn.var_length)]"  # outputs "123"
 
 concat([<start>],[<var>],[<end>])
   Concatenates up to 3 fields after the current sample which is then turned to

diff --git a/reg-tests/converter/bytes.vtc b/reg-tests/converter/bytes.vtc
new file mode 100644 (file)
index 0000000..8abe401
--- /dev/null
+++ b/reg-tests/converter/bytes.vtc
@@ -0,0 +1,156 @@
+varnishtest "bytes converter Test"
+
+feature cmd "$HAPROXY_PROGRAM -cc 'version_atleast(2.9-dev4)'"
+
+feature ignore_unknown_macro
+
+# TEST - 1
+server s1 {
+       rxreq
+       txresp -hdr "Connection: close"
+} -repeat 1 -start
+
+haproxy h1 -conf {
+       defaults
+               mode http
+               timeout connect "${HAPROXY_TEST_TIMEOUT-5s}"
+               timeout client  "${HAPROXY_TEST_TIMEOUT-5s}"
+               timeout server  "${HAPROXY_TEST_TIMEOUT-5s}"
+
+       frontend fe
+               bind "fd@${fe}"
+
+               #### requests
+               http-request  set-var(txn.input) req.hdr(input)
+
+               http-response set-header bytes_0      "%[var(txn.input),bytes(0)]"
+               http-response set-header bytes_1      "%[var(txn.input),bytes(1)]"
+               http-response set-header bytes_0_3    "%[var(txn.input),bytes(0,3)]"
+               http-response set-header bytes_1_3    "%[var(txn.input),bytes(1,3)]"
+               http-response set-header bytes_99     "%[var(txn.input),bytes(99)]"
+               http-response set-header bytes_5      "%[var(txn.input),bytes(5)]"
+               http-response set-header bytes_6      "%[var(txn.input),bytes(6)]"
+               http-response set-header bytes_0_6    "%[var(txn.input),bytes(0,6)]"
+               http-response set-header bytes_0_7    "%[var(txn.input),bytes(0,7)]"
+
+               http-response set-var(txn.var_start) int(0)
+               http-response set-header bytes_var0    "%[var(txn.input),bytes(txn.var_start)]"
+
+               http-response set-var(txn.var_start) int(1)
+               http-response set-var(txn.var_length) int(3)
+               http-response set-header bytes_var1_var3    "%[var(txn.input),bytes(txn.var_start,txn.var_length)]"
+
+               http-response set-var(txn.var_start) int(99)
+               http-response set-header bytes_var99  "%[var(txn.input),bytes(txn.var_start)]"
+
+               http-response set-var(txn.var_start) int(0)
+               http-response set-var(txn.var_length) int(7)
+               http-response set-header bytes_var0_var7    "%[var(txn.input),bytes(txn.var_start,txn.var_length)]"
+
+               http-response set-var(txn.var_start) int(1)
+               http-response set-var(txn.var_length) int(3)
+               http-response set-header bytes_var1_3   "%[var(txn.input),bytes(txn.var_start,3)]"
+               http-response set-header bytes_1_var3   "%[var(txn.input),bytes(1,txn.var_length)]"
+
+               http-response set-var(txn.var_start) int(-1)
+               http-response set-var(txn.var_length) int(-1)
+               http-response set-header bytes_varminus1        "%[var(txn.input),bytes(txn.var_start)]"
+               http-response set-header bytes_0_varminus1      "%[var(txn.input),bytes(0,txn.var_length)]"
+
+               http-response set-header bytes_varNA    "%[var(txn.input),bytes(txn.NA)]"
+               http-response set-header bytes_1_varNA  "%[var(txn.input),bytes(1,txn.NA)]"
+
+               default_backend be
+
+       backend be
+               server s1 ${s1_addr}:${s1_port}
+} -start
+
+client c1 -connect ${h1_fe_sock} {
+       txreq -url "/" \
+               -hdr "input: 012345"
+       rxresp
+       expect resp.status == 200
+       expect resp.http.bytes_0 == "012345"
+       expect resp.http.bytes_1 == "12345"
+       expect resp.http.bytes_0_3 == "012"
+       expect resp.http.bytes_1_3 == "123"
+       expect resp.http.bytes_99 == ""
+       expect resp.http.bytes_5 == "5"
+       expect resp.http.bytes_6 == ""
+       expect resp.http.bytes_0_6 == "012345"
+
+       # since specified length is > input length, response contains the input till the end
+       expect resp.http.bytes_0_7 == "012345"
+
+       expect resp.http.bytes_var0 == "012345"
+       expect resp.http.bytes_var1_var3 == "123"
+       expect resp.http.bytes_var99 == ""
+       expect resp.http.bytes_var0_var7 == "012345"
+       expect resp.http.bytes_var1_3 == "123"
+       expect resp.http.bytes_1_var3 == "123"
+       expect resp.http.bytes_varminus1 == ""
+       expect resp.http.bytes_0_varminus1 == ""
+       expect resp.http.bytes_varNA == ""
+       expect resp.http.bytes_1_varNA == ""
+} -run
+
+# TEST - 2
+# negative starting index causes startup failure
+haproxy h2 -conf {
+       defaults
+               mode http
+               timeout connect "${HAPROXY_TEST_TIMEOUT-5s}"
+               timeout client  "${HAPROXY_TEST_TIMEOUT-5s}"
+               timeout server  "${HAPROXY_TEST_TIMEOUT-5s}"
+
+       frontend fe
+               bind "fd@${fe}"
+
+               http-response set-header bytes_output     "%[var(txn.input),bytes(-1)]"
+
+               default_backend be
+
+       backend be
+               server s1 ${s1_addr}:${s1_port}
+} -start -expectexit 1
+
+# TEST - 3
+# negative length causes startup failure
+haproxy h3 -conf {
+       defaults
+               mode http
+               timeout connect "${HAPROXY_TEST_TIMEOUT-5s}"
+               timeout client  "${HAPROXY_TEST_TIMEOUT-5s}"
+               timeout server  "${HAPROXY_TEST_TIMEOUT-5s}"
+
+       frontend fe
+               bind "fd@${fe}"
+
+               http-response set-header bytes_output     "%[var(txn.input),bytes(0,-1)]"
+
+               default_backend be
+
+       backend be
+               server s1 ${s1_addr}:${s1_port}
+} -start -expectexit 1
+
+# TEST - 4
+# 0 length causes startup failure
+haproxy h4 -conf {
+       defaults
+               mode http
+               timeout connect "${HAPROXY_TEST_TIMEOUT-5s}"
+               timeout client  "${HAPROXY_TEST_TIMEOUT-5s}"
+               timeout server  "${HAPROXY_TEST_TIMEOUT-5s}"
+
+       frontend fe
+               bind "fd@${fe}"
+
+               http-response set-header bytes_output     "%[var(txn.input),bytes(0,0)]"
+
+               default_backend be
+
+       backend be
+               server s1 ${s1_addr}:${s1_port}
+} -start -expectexit 1

diff --git a/src/sample.c b/src/sample.c
index 07c881dcf7bdf1f77873ce744fb191904ea2e9f6..bb7db28e797798e17f27b9f0f8e0bd360b78f65c 100644 (file)
--- a/src/sample.c
+++ b/src/sample.c
@@ -2705,18 +2705,54 @@ static int sample_conv_json(const struct arg *arg_p, struct sample *smp, void *p
  * Optional second arg is the length to truncate */
 static int sample_conv_bytes(const struct arg *arg_p, struct sample *smp, void *private)
 {
-       if (smp->data.u.str.data <= arg_p[0].data.sint) {
+       struct sample smp_arg0, smp_arg1;
+       long long start_idx, length;
+
+       // determine the start_idx and length of the output
+       smp_set_owner(&smp_arg0, smp->px, smp->sess, smp->strm, smp->opt);
+       if (!sample_conv_var2smp_sint(&arg_p[0], &smp_arg0)) {
                smp->data.u.str.data = 0;
-               return 1;
+               return 0;
        }
+       if (smp_arg0.data.u.sint < 0 || (smp_arg0.data.u.sint >= smp->data.u.str.data)) {
+               // empty output if the arg0 value is negative or >= the input length
+               smp->data.u.str.data = 0;
+               return 0;
+       }
+       start_idx = smp_arg0.data.u.sint;
 
-       if (smp->data.u.str.size)
-                       smp->data.u.str.size -= arg_p[0].data.sint;
-       smp->data.u.str.data -= arg_p[0].data.sint;
-       smp->data.u.str.area += arg_p[0].data.sint;
+       // length comes from arg1 if present, otherwise it's the remaining length
+       if (arg_p[1].type != ARGT_STOP) {
+               smp_set_owner(&smp_arg1, smp->px, smp->sess, smp->strm, smp->opt);
+               if (!sample_conv_var2smp_sint(&arg_p[1], &smp_arg1)) {
+                       smp->data.u.str.data = 0;
+                       return 0;
+               }
+               if (smp_arg1.data.u.sint < 0) {
+                       // empty output if the arg1 value is negative
+                       smp->data.u.str.data = 0;
+                       return 0;
+               }
+
+               if (smp_arg1.data.u.sint > (smp->data.u.str.data - start_idx)) {
+                       // arg1 value is greater than the remaining length
+                       if (smp->opt & SMP_OPT_FINAL) {
+                               // truncate to remaining length
+                               length = smp->data.u.str.data - start_idx;
+                       } else {
+                               smp->data.u.str.data = 0;
+                               return 0;
+                       }
+               } else {
+                       length = smp_arg1.data.u.sint;
+               }
+       } else {
+               length = smp->data.u.str.data - start_idx;
+       }
 
-       if ((arg_p[1].type == ARGT_SINT) && (arg_p[1].data.sint < smp->data.u.str.data))
-               smp->data.u.str.data = arg_p[1].data.sint;
+       // update the output using the start_idx and length
+       smp->data.u.str.area += start_idx;
+       smp->data.u.str.data = length;
 
        return 1;
 }
@@ -4929,6 +4965,34 @@ static int smp_fetch_bytes(const struct arg *args, struct sample *smp, const cha
        return 1;
 }
 
+static int sample_conv_bytes_check(struct arg *args, struct sample_conv *conv,
+                          const char *file, int line, char **err)
+{
+       // arg0 is not optional, must be >= 0
+       if (!check_operator(&args[0], conv, file, line, err)) {
+               return 0;
+       }
+       if (args[0].type != ARGT_VAR) {
+               if (args[0].type != ARGT_SINT || args[0].data.sint < 0) {
+                       memprintf(err, "expects a non-negative integer");
+                       return 0;
+               }
+       }
+       // arg1 is optional, must be > 0
+       if (args[1].type != ARGT_STOP) {
+               if (!check_operator(&args[1], conv, file, line, err)) {
+                       return 0;
+               }
+               if (args[1].type != ARGT_VAR) {
+                       if (args[1].type != ARGT_SINT || args[1].data.sint <= 0) {
+                               memprintf(err, "expects a positive integer");
+                               return 0;
+                       }
+               }
+       }
+
+       return 1;
+}
 
 static struct sample_fetch_kw_list smp_logs_kws = {ILH, {
        { "bytes_in",             smp_fetch_bytes,        0,         NULL, SMP_T_SINT, SMP_USE_INTRN },
@@ -5025,7 +5089,7 @@ static struct sample_conv_kw_list sample_conv_kws = {ILH, {
        { "xxh32",   sample_conv_xxh32,        ARG1(0,SINT),          NULL,                     SMP_T_BIN,  SMP_T_SINT },
        { "xxh64",   sample_conv_xxh64,        ARG1(0,SINT),          NULL,                     SMP_T_BIN,  SMP_T_SINT },
        { "json",    sample_conv_json,         ARG1(1,STR),           sample_conv_json_check,   SMP_T_STR,  SMP_T_STR  },
-       { "bytes",   sample_conv_bytes,        ARG2(1,SINT,SINT),     NULL,                     SMP_T_BIN,  SMP_T_BIN  },
+       { "bytes",   sample_conv_bytes,        ARG2(1,STR,STR),       sample_conv_bytes_check,  SMP_T_BIN,  SMP_T_BIN  },
        { "field",   sample_conv_field,        ARG3(2,SINT,STR,SINT), sample_conv_field_check,  SMP_T_STR,  SMP_T_STR  },
        { "word",    sample_conv_word,         ARG3(2,SINT,STR,SINT), sample_conv_field_check,  SMP_T_STR,  SMP_T_STR  },
        { "param",   sample_conv_param,        ARG2(1,STR,STR),       sample_conv_param_check,  SMP_T_STR,  SMP_T_STR  },