add support for foreach ... (%sql("SELECT ...))

diff --git a/doc/antora/modules/reference/pages/unlang/foreach.adoc b/doc/antora/modules/reference/pages/unlang/foreach.adoc
index 8b2caaecccea72b3a91ee686b6fcc20597fd33d1..1d11dbcb3f92b117dca1755238f92c054d872216 100644 (file)
--- a/doc/antora/modules/reference/pages/unlang/foreach.adoc
+++ b/doc/antora/modules/reference/pages/unlang/foreach.adoc
@@ -3,7 +3,7 @@
 .Syntax
 [source,unlang]
 ----
-foreach [<data-type>] <key-name> (<attribute-reference>) {
+foreach [<data-type>] <key-name> (<reference>) {
     [ statements ]
 }
 ----
@@ -28,14 +28,24 @@ The `<key-name>` can be modified during the course of the `foreach` loop.  Modif
 
 The only limitation on the `<key-name>` is that it must be unique.
 
-<attribute-reference>::
+<reference>::
 
-The xref:unlang/attr.adoc[attribute reference] which will will be looped
+An xref:unlang/attr.adoc[attribute reference] which will will be looped
 over.  The reference can be to one attribute, to an array, a child, or
 be a subset of attributes.
 
+Alternatively, the `<reference>` can be a xref:reference:xlat/index.adoc[dynamic expansion function],
+such as `%sql("SELECT ...")`.  When the reference is a dynamic
+expansion function, a `<data-type>` must be specified.
+
 == Modifying Loop variables
 
+When the `<reference>` is an attribute, the attribute being looped
+over can sometimes be modified.  When the `<reference>` is a dynamic
+expansion, the results cannot be modified, and are discarded when the
+`foreach` loop is finished.  If it is necessary to save the results,
+they should be placed into another attribute.
+
 An attribute which is a "leaf" data type (e.g. `uint32`, and not
 `tlv`) will be automatically copied back to the original attribute at
 the end of each iteration of the `foreach` loop.  That is, the

diff --git a/src/lib/unlang/compile.c b/src/lib/unlang/compile.c
index 09e4b1a6e43960868f2a55dd97f7620f5025c425..4a78d3ee81d402e711a94d6f66c21a94bb3d9b1e 100644 (file)
--- a/src/lib/unlang/compile.c
+++ b/src/lib/unlang/compile.c
@@ -3225,6 +3225,11 @@ static unlang_t *compile_foreach(unlang_t *parent, unlang_compile_t *unlang_ctx,
         *      will fix it up.
         */
        token = cf_section_name2_quote(cs);
+       if (token != T_BARE_WORD) {
+               cf_log_err(cs, "Data being looped over in 'foreach' must be an attribute reference or dynamic expansion, not a string");
+               goto fail;
+       }
+
        slen = tmpl_afrom_substr(g, &vpt,
                                 &FR_SBUFF_IN(name2, strlen(name2)),
                                 token,
@@ -3243,19 +3248,19 @@ static unlang_t *compile_foreach(unlang_t *parent, unlang_compile_t *unlang_ctx,
         */
        fr_assert(vpt);
 
-       if (!tmpl_is_attr(vpt)) {
-               cf_log_err(cs, "MUST use attribute or list reference (not %s) in 'foreach'",
-                          tmpl_type_to_str(vpt->type));
-               goto fail;
-       }
+       if (tmpl_is_attr(vpt)) {
+               if (tmpl_attr_tail_num(vpt) == NUM_UNSPEC) {
+                       cf_log_warn(cs, "Attribute reference should be updated to use %s[*]", vpt->name);
+                       tmpl_attr_rewrite_leaf_num(vpt, NUM_ALL);
+               }
 
-       if (tmpl_attr_tail_num(vpt) == NUM_UNSPEC) {
-               cf_log_warn(cs, "Attribute reference should be updated to use %s[*]", vpt->name);
-               tmpl_attr_rewrite_leaf_num(vpt, NUM_ALL);
-       }
+               if (tmpl_attr_tail_num(vpt) != NUM_ALL) {
+                       cf_log_err(cs, "MUST NOT use instance selectors in 'foreach'");
+                       goto fail;
+               }
 
-       if (tmpl_attr_tail_num(vpt) != NUM_ALL) {
-               cf_log_err(cs, "MUST NOT use instance selectors in 'foreach'");
+       } else if (!tmpl_contains_xlat(vpt)) {
+               cf_log_err(cs, "Invalid contents in 'foreach (...)', it must be an attribute reference or a dynamic expansion");
                goto fail;
        }
 
@@ -3284,6 +3289,25 @@ static unlang_t *compile_foreach(unlang_t *parent, unlang_compile_t *unlang_ctx,
         *      If we have "type name", then define a local variable of that name.
         */
        type_name = cf_section_argv(cs, 0); /* AFTER name1, name2 */
+
+       if (tmpl_is_xlat(vpt)) {
+               if (!type_name) {
+                       /*
+                        *      @todo - find a way to get the data type.
+                        */
+                       cf_log_err(cs, "Dynamic expansions MUST specify a data type for the variable");
+                       return NULL;
+               }
+
+               type = fr_table_value_by_str(fr_type_table, type_name, FR_TYPE_VOID);
+               if (!fr_type_is_leaf(type)) {
+                       cf_log_err(cs, "Dynamic expansions MUST specify a non-structural data type for the variable");
+                       return NULL;
+               }
+
+               goto get_name;
+       }
+
        if (type_name) {
                unlang_variable_t *var;
                fr_dict_attr_t const *da = tmpl_attr_tail_da(vpt);
@@ -3304,6 +3328,7 @@ static unlang_t *compile_foreach(unlang_t *parent, unlang_compile_t *unlang_ctx,
                        goto incompatible;
                }
 
+       get_name:
                variable_name = cf_section_argv(cs, 1);
 
                /*

diff --git a/src/lib/unlang/foreach.c b/src/lib/unlang/foreach.c
index b80573f8d6ca74237887d74711f48522d73b4dc2..f4dea812cea8ed6ed5365fc928a114f3a3cbf004 100644 (file)
--- a/src/lib/unlang/foreach.c
+++ b/src/lib/unlang/foreach.c
@@ -54,6 +54,9 @@ typedef struct {
        fr_pair_t               *key;                           //!< local variable which contains the key
        tmpl_t const            *vpt;                           //!< pointer to the vpt
 
+       bool                    success;                        //!< for xlat expansion
+       fr_value_box_list_t     list;                           //!< value box list for looping over xlats
+
        tmpl_dcursor_ctx_t      cc;                             //!< tmpl cursor state
 
                                                                ///< we're iterating over.
@@ -132,11 +135,11 @@ static int unlang_foreach_pair_copy(fr_pair_t *to, fr_pair_t *from, fr_dict_attr
        return 0;
 }
 
-static xlat_action_t unlang_foreach_xlat(TALLOC_CTX *ctx, fr_dcursor_t *out,
-                                        xlat_ctx_t const *xctx,
-                                        request_t *request, UNUSED fr_value_box_list_t *in);
+static xlat_action_t unlang_foreach_xlat_func(TALLOC_CTX *ctx, fr_dcursor_t *out,
+                                             xlat_ctx_t const *xctx,
+                                             request_t *request, UNUSED fr_value_box_list_t *in);
 
-#define FOREACH_REQUEST_DATA (void *)unlang_foreach_xlat
+#define FOREACH_REQUEST_DATA (void *)unlang_foreach_xlat_func
 
 /** Ensure request data is pulled out of the request if the frame is popped
  *
@@ -146,6 +149,8 @@ static int _free_unlang_frame_state_foreach(unlang_frame_state_foreach_t *state)
        if (state->key) {
                fr_pair_t *vp;
 
+               if (tmpl_is_xlat(state->vpt)) return 0;
+
                tmpl_dcursor_clear(&state->cc);
 
                /*
@@ -210,8 +215,94 @@ static unlang_action_t unlang_foreach_next_old(rlm_rcode_t *p_result, request_t
        return unlang_interpret_push_children(p_result, request, frame->result, UNLANG_NEXT_SIBLING);
 }
 
+static unlang_action_t unlang_foreach_xlat_next(rlm_rcode_t *p_result, request_t *request, unlang_stack_frame_t *frame)
+{
+       unlang_frame_state_foreach_t    *state = talloc_get_type_abort(frame->state, unlang_frame_state_foreach_t);
+       fr_value_box_t *box;
+
+next:
+       box = fr_dcursor_next(&state->cursor);
+       if (!box) {
+               *p_result = frame->result;
+               return UNLANG_ACTION_CALCULATE_RESULT;
+       }
+
+       fr_value_box_clear_value(&state->key->data);
+       if (fr_value_box_cast(state->key, &state->key->data, state->key->vp_type, state->key->da, box) < 0) {
+               RDEBUG("Failed casting 'foreach' iteration variable '%s' from %pV", state->key->da->name, box);
+               goto next;
+       }
+
+       repeatable_set(frame);
+
+       /*
+        *      Push the child, and yield for a later return.
+        */
+       return unlang_interpret_push_children(p_result, request, frame->result, UNLANG_NEXT_SIBLING);
+}
 
-static unlang_action_t unlang_foreach_next(rlm_rcode_t *p_result, request_t *request, unlang_stack_frame_t *frame)
+
+static unlang_action_t unlang_foreach_xlat_expanded(rlm_rcode_t *p_result, request_t *request, unlang_stack_frame_t *frame)
+{
+       unlang_frame_state_foreach_t    *state = talloc_get_type_abort(frame->state, unlang_frame_state_foreach_t);
+       fr_value_box_t *box;
+
+       if (!state->success) {  
+               RDEBUG("Failed expanding 'foreach' list");
+               *p_result = RLM_MODULE_FAIL;
+               return UNLANG_ACTION_CALCULATE_RESULT;
+       }
+
+       box = fr_dcursor_init(&state->cursor, fr_value_box_list_dlist_head(&state->list));
+       if (!box) {
+       done:
+               *p_result = RLM_MODULE_NOOP;
+               return UNLANG_ACTION_CALCULATE_RESULT;
+       }
+
+       fr_value_box_clear_value(&state->key->data);
+
+next:
+       if (fr_value_box_cast(state->key, &state->key->data, state->key->vp_type, state->key->da, box) < 0) {
+               RDEBUG("Failed casting 'foreach' iteration variable '%s' from %pV", state->key->da->name, box);
+               box = fr_dcursor_next(&state->cursor);
+               if (!box) goto done;
+
+               goto next;
+       }
+
+       frame->process = unlang_foreach_xlat_next;
+       repeatable_set(frame);
+
+       /*
+        *      Push the child, and yield for a later return.
+        */
+       return unlang_interpret_push_children(p_result, request, frame->result, UNLANG_NEXT_SIBLING);
+}
+
+
+/*
+ *     Loop over an xlat expansion
+ */
+static unlang_action_t unlang_foreach_xlat_init(rlm_rcode_t *p_result, request_t *request, unlang_stack_frame_t *frame,
+                                               unlang_frame_state_foreach_t *state)
+{
+       fr_value_box_list_init(&state->list);
+
+       if (unlang_xlat_push(state, &state->success, &state->list, request, tmpl_xlat(state->vpt), false) < 0) {
+               REDEBUG("Failed starting expansion of %s", state->vpt->name);
+               *p_result = RLM_MODULE_FAIL;
+               return UNLANG_ACTION_CALCULATE_RESULT;
+       }
+
+       frame->process = unlang_foreach_xlat_expanded;
+       repeatable_set(frame);
+
+       return UNLANG_ACTION_PUSHED_CHILD;
+}
+
+
+static unlang_action_t unlang_foreach_attr_next(rlm_rcode_t *p_result, request_t *request, unlang_stack_frame_t *frame)
 {
        unlang_frame_state_foreach_t    *state = talloc_get_type_abort(frame->state, unlang_frame_state_foreach_t);
        fr_pair_t                       *vp;
@@ -292,6 +383,80 @@ next:
        return unlang_interpret_push_children(p_result, request, frame->result, UNLANG_NEXT_SIBLING);
 }
 
+/*
+ *     Loop over an attribute
+ */
+static unlang_action_t unlang_foreach_attr_init(rlm_rcode_t *p_result, request_t *request, unlang_stack_frame_t *frame,
+                                               unlang_frame_state_foreach_t *state)
+{
+       fr_pair_t                       *vp;
+
+       /*
+        *      No matching attributes, we can't do anything.
+        */
+       vp = tmpl_dcursor_init(NULL, NULL, &state->cc, &state->cursor, request, state->vpt);
+       if (!vp) {
+               *p_result = RLM_MODULE_NOOP;
+               return UNLANG_ACTION_CALCULATE_RESULT;
+       }
+
+       /*
+        *      Before we loop over the variables, ensure that the user can't pull the rug out from
+        *      under us.
+        */
+       do {
+               if (fr_type_is_leaf(vp->vp_type)) fr_pair_set_immutable(vp);
+               
+       } while ((vp = fr_dcursor_next(&state->cursor)) != NULL);
+       tmpl_dcursor_clear(&state->cc);
+
+       vp = tmpl_dcursor_init(NULL, NULL, &state->cc, &state->cursor, request, state->vpt);
+       fr_assert(vp != NULL);
+
+       if (vp->vp_type == FR_TYPE_GROUP) {
+               fr_assert(state->key->vp_type == FR_TYPE_GROUP);
+
+               if (fr_pair_list_copy(state->key, &state->key->vp_group, &vp->vp_group) < 0) {
+                       REDEBUG("Failed copying members of %s", state->key->da->name);
+                       *p_result = RLM_MODULE_FAIL;
+                       return UNLANG_ACTION_CALCULATE_RESULT;
+               }
+
+       } else if (fr_type_is_structural(vp->vp_type)) {
+               fr_assert(state->key->vp_type == vp->vp_type);
+
+               if (unlang_foreach_pair_copy(state->key, vp, vp->da) < 0) {
+                       REDEBUG("Failed copying children of %s", state->key->da->name);
+                       *p_result = RLM_MODULE_FAIL;
+                       return UNLANG_ACTION_CALCULATE_RESULT;
+               }
+
+       } else {
+               fr_value_box_clear_value(&state->key->data);
+               while (vp && (fr_value_box_cast(state->key, &state->key->data, state->key->vp_type, state->key->da, &vp->data) < 0)) {
+                       RDEBUG("Failed casting 'foreach' iteration variable '%s' from %pP", state->key->da->name, vp);
+                       vp = fr_dcursor_next(&state->cursor);
+               }
+
+               /*
+                *      Couldn't cast anything, the loop can't be run.
+                */
+               if (!vp) {
+                       *p_result = RLM_MODULE_NOOP;
+                       return UNLANG_ACTION_CALCULATE_RESULT;
+               }
+       }
+
+       frame->process = unlang_foreach_attr_next;
+
+       repeatable_set(frame);
+
+       /*
+        *      Push the child, and go process it.
+        */
+       return unlang_interpret_push_children(p_result, request, frame->result, UNLANG_NEXT_SIBLING);
+}
+
 
 static unlang_action_t unlang_foreach(rlm_rcode_t *p_result, request_t *request, unlang_stack_frame_t *frame)
 {
@@ -325,28 +490,6 @@ static unlang_action_t unlang_foreach(rlm_rcode_t *p_result, request_t *request,
        if (gext->key) {
                state->vpt = gext->vpt;
 
-               /*
-                *      No matching attributes, we can't do anything.
-                */
-               vp = tmpl_dcursor_init(NULL, NULL, &state->cc, &state->cursor, request, gext->vpt);
-               if (!vp) {
-                       *p_result = RLM_MODULE_NOOP;
-                       return UNLANG_ACTION_CALCULATE_RESULT;
-               }
-
-               /*
-                *      Before we loop over the variables, ensure that the user can't pull the rug out from
-                *      under us.
-                */
-               do {
-                       if (fr_type_is_leaf(vp->vp_type)) fr_pair_set_immutable(vp);
-               
-               } while ((vp = fr_dcursor_next(&state->cursor)) != NULL);
-               tmpl_dcursor_clear(&state->cc);
-
-               vp = tmpl_dcursor_init(NULL, NULL, &state->cc, &state->cursor, request, gext->vpt);
-               fr_assert(vp != NULL);
-
                /*
                 *      Create the local variable and populate its value.
                 */
@@ -357,48 +500,13 @@ static unlang_action_t unlang_foreach(rlm_rcode_t *p_result, request_t *request,
                }
                fr_assert(state->key != NULL);
 
-               if (vp->vp_type == FR_TYPE_GROUP) {
-                       fr_assert(state->key->vp_type == FR_TYPE_GROUP);
-
-                       if (fr_pair_list_copy(state->key, &state->key->vp_group, &vp->vp_group) < 0) {
-                               REDEBUG("Failed copying members of %s", gext->key->name);
-                               *p_result = RLM_MODULE_FAIL;
-                               return UNLANG_ACTION_CALCULATE_RESULT;
-                       }
-
-               } else if (fr_type_is_structural(vp->vp_type)) {
-                       fr_assert(state->key->vp_type == vp->vp_type);
-
-                       if (unlang_foreach_pair_copy(state->key, vp, vp->da) < 0) {
-                               REDEBUG("Failed copying children of %s", state->key->da->name);
-                               *p_result = RLM_MODULE_FAIL;
-                               return UNLANG_ACTION_CALCULATE_RESULT;
-                       }
-
-               } else {
-                       fr_value_box_clear_value(&state->key->data);
-                       while (vp && (fr_value_box_cast(state->key, &state->key->data, state->key->vp_type, state->key->da, &vp->data) < 0)) {
-                               RDEBUG("Failed casting 'foreach' iteration variable '%s' from %pP", state->key->da->name, vp);
-                               vp = fr_dcursor_next(&state->cursor);
-                       }
-
-                       /*
-                        *      Couldn't cast anything, the loop can't be run.
-                        */
-                       if (!vp) {
-                               *p_result = RLM_MODULE_NOOP;
-                               return UNLANG_ACTION_CALCULATE_RESULT;
-                       }
+               if (tmpl_is_attr(gext->vpt)) {
+                       return unlang_foreach_attr_init(p_result, request, frame, state);
                }
 
-               frame->process = unlang_foreach_next;
-
-               repeatable_set(frame);
+               fr_assert(tmpl_is_xlat(gext->vpt));
 
-               /*
-                *      Push the child, and go process it.
-                */
-               return unlang_interpret_push_children(p_result, request, frame->result, UNLANG_NEXT_SIBLING);
+               return unlang_foreach_xlat_init(p_result, request, frame, state);
        }
 
        /*
@@ -493,9 +601,9 @@ static unlang_action_t unlang_break(rlm_rcode_t *p_result, request_t *request, u
  *
  * @ingroup xlat_functions
  */
-static xlat_action_t unlang_foreach_xlat(TALLOC_CTX *ctx, fr_dcursor_t *out,
-                                        xlat_ctx_t const *xctx,
-                                        request_t *request, UNUSED fr_value_box_list_t *in)
+static xlat_action_t unlang_foreach_xlat_func(TALLOC_CTX *ctx, fr_dcursor_t *out,
+                                             xlat_ctx_t const *xctx,
+                                             request_t *request, UNUSED fr_value_box_list_t *in)
 {
        fr_pair_t                       *vp;
        int const                       *inst = xctx->inst;
@@ -522,7 +630,7 @@ void unlang_foreach_init(TALLOC_CTX *ctx)
                xlat_t *x;
 
                x = xlat_func_register(ctx, xlat_foreach_names[i],
-                                 unlang_foreach_xlat, FR_TYPE_VOID);
+                                 unlang_foreach_xlat_func, FR_TYPE_VOID);
                fr_assert(x);
                xlat_func_flags_set(x, XLAT_FUNC_FLAG_INTERNAL);
                x->uctx = &xlat_foreach_inst[i];

diff --git a/src/tests/keywords/foreach-explode b/src/tests/keywords/foreach-explode
new file mode 100644 (file)
index 0000000..ce4b1be
--- /dev/null
+++ b/src/tests/keywords/foreach-explode
@@ -0,0 +1,21 @@
+#
+#  PRE: foreach xlat-explode
+#
+string input
+string result
+
+input = "a b c d e f"
+result = ""
+
+foreach string thing (%explode(%{input}, ' ')) {
+       result += thing
+       result += ','
+}
+
+result -= ','
+
+if (result != "a,b,c,d,e,f") {
+       test_fail
+}
+
+success