From: Alan T. DeKok Date: Wed, 7 Aug 2019 14:42:25 +0000 (-0400) Subject: add "filter" keyword X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=118f59cc2baa3935ea3618ff6cf60bdc6f103dec;p=thirdparty%2Ffreeradius-server.git add "filter" keyword and document it. And add tests for it. And note changes in the "upgrade" documentation --- diff --git a/doc/unlang/filter.adoc b/doc/unlang/filter.adoc new file mode 100644 index 00000000000..a4bdbd031fd --- /dev/null +++ b/doc/unlang/filter.adoc @@ -0,0 +1,131 @@ +== The filter Statement + +.Syntax +[source,unlang] +---- +filter [ ] { + + ... +} +---- + +The `filter` statement filters attributes in, the named __. +Attributes which do not match the filter conditions are removed. +Attributes that are not named in the `filter` section are unchanged. + +The `filter` statement consists of the following syntax elements: + +:: The attribute list which will be filtered. The list is +usually `request`, `reply`, or `control`. ++ +If the __ qualifier is omitted, then each entry inside of the +`filter` section *must* be prefixed with a list name. For example, +`&request:User-Name ...` + +:: The server attribute which is being filtered via the given +__. + +:: The operator such as `<`, `>=`, etc. + +:: The value which is used to filter the attribute. If the +field is a double-quoted string, it undergoes link:xlat.adoc[string +expansion], and the resulting value is used to filter the attribute. + +The change process is atomic, in that either all of the attributes are +modified, or none of them are modified. If the `filter` fails for any +reason, then all of the results are discarded, and the `filter` does +not affect any server attributes. + +.Example +[source,unlang] +---- +filter reply { + &Session-Timeout <= 3600 +} +---- + +=== Lists + +The __ field sets the attribute list that will be filter. If +the __ qualifier is omitted, then each entry inside of the +`filter` section *must* be prefixed with a list name. For example, +`&request:User-Name ...` + +Please see the link:list.adoc[list] page for valid list names. + +=== Server Attributes + +The __ field is an attribute name, such as +`&Reply-Message`. The attribute name may also be prefixed with a +__ qualifier, which overrides the __ given at the start +of the `filter` section. + +NOTE: In version 3, the leading `&` was optional. In version 4, the +leading `&` is required. + +=== Editing Operators + +The `` field is used to define how the values are filtered +Different operators allow attributes to be added, deleted, or +replaced, as defined below. + +=== Filtering Operators + +The following operators may also be used in addition to the ones +listed above. These operators use the __ and +__ fields to enforce limits on all attributes in the given +__, and to edit attributes which have a matching +__ name. All other attributes are ignored. + +.Filtering Operators +[options="header] +[cols="10%,90%"] +|===== +| Operator | Description +| -= | Remove all attributes from the list that match __. +| == | Keep only the attributes in the list that match __ +| < | Keep only the attributes in the list that have values less than __. +| \<= | Keep only the attributes in the list that have values less than or equal to __. +| > | Keep only the attributes in the list that have values greater than __. +| >= | Keep only the attributes in the list that have values greater than or equal to __. +| !* | Delete all occurances of the attribute, no matter what the value. +| =~ | Keep only the attributes in the list which match the regular expression given in __. +| !~ | Keep only the attributes in the list which do not match the regular expression given in __. +|===== + +The `==` operator is very different from the `=` operator listed +above. The `=` operator is used to add new attributes to the list, +while the `==` operator removes all attributes that do not match the +given value. + +For IP addresses, the operators `>`, `>=`, `<`, and `<=` check for +membership in a network. The __ field should then be a IP +network, given in `address/mask` format. + +IMPORTANT: In version 3, some filtering operators would _create_ the +attribute if it did not exist in the destination list. In version 4, +this functionality has been removed. Instead, version 4 only performs +_filtering_ of the attributes. That is, the filtering operators will +modify or delete attributes, but they will never create an attribute. + +=== Values + +The __ field is the value which is used to filter the +__. The interpretation of the __ field +depends on the data type of the contents. For example, if the string +`"192.0.2.1"` is assigned to an attribute of the `string` data type, +then the result is an ASCII string containing that value. However, if +the same string is assigned to an attribute of the `ipaddr` data type, +then the result is a 32-bit IPv4 address, with binary value +`0xc0000201`. + +.Example +[source,unlang] +---- +filter reply { + &Session-Timeout <= 3600 +} +---- + +// Copyright (C) 2019 Network RADIUS SAS. Licenced under CC-by-NC 4.0. +// Development of this documentation was sponsored by Network RADIUS SAS. diff --git a/doc/unlang/keywords.adoc b/doc/unlang/keywords.adoc index dc504088601..1c30ee2c083 100644 --- a/doc/unlang/keywords.adoc +++ b/doc/unlang/keywords.adoc @@ -32,8 +32,9 @@ modify attributes in any list or packet. [options="header"] |===== | Keyword | Description +| link:filter.adoc[filter] | Filter attributes from a list | link:map.adoc[map] | Map database fields to server attributes. -| link:update.adoc[update] | Edit a list of attributes. +| link:update.adoc[update] | Add attributes to a list |===== === Grouping Keywords diff --git a/doc/unlang/list.adoc b/doc/unlang/list.adoc new file mode 100644 index 00000000000..083fd227596 --- /dev/null +++ b/doc/unlang/list.adoc @@ -0,0 +1,55 @@ +== Attribute Lists + +An attribute list contains a set of attributes. The allowed lists +are: + +`request`:: Attributes in the incoming request packet. + +`reply`:: Attributes in the outgoing reply packet. + +`control`:: Attributes in the internal "control" list that is +associated with the request. ++ +The `control` attributes are used to manage how the request is +processed. These attributes are never sent in any packet. + +`session-state`:: Attributes which are maintained across multi-packet +exchanges. + +There must be a colon `:` after the list name and before the attribute name. +This syntax helps the server to distinguish between list names and attribute +names. + +NOTE: In version 3, there were additional lists such as `proxy`, +`proxy-reply`, and `coa`. These lists have been removed in version 4. +See the link:subrequest.adoc[subrequest] statement for how to create +child requests. + +With the exception of `session-state`, all of the above lists are +ephemeral. That is, they exist for one packet exchange, and only one +packet exchange. When a reply is sent for a request, the above lists +and all attributes are deleted. There is no way to reference an +attribute from a previous packet. We recommend using a database to +track complex state. + +In some cases, requests are associated a multi-packet exchange. For +those situations, the `session-state` list is automatically saved when +a reply is sent, and it is automatically restored when the next packet +in sequence comes in. Once the packet exchange has been finished, the +`session-state` list is deleted. + +In some cases, there is a parent-child relationship between requests. +In those situations, it is possible for the policy rules in the child +to refer to attributes in the parent. This reference can be made by +prefixing the __ name with the `parent` qualifier. The key word +`outer` is also a synonym for `parent`. If there are multiple +parent-child relationships, the `parent` qualifier can be repeated. + +There is, however, no way for the parent to refer to the child. When +the child is running, the parent is suspended. Once the child +finishes, it is deleted, and is no longer accessible to the parent. + +.Examples +* `&parent.request:User-Name` +* `&parent.reply:Reply-Message` +* `&parent.parent.session-state:Filter-Id` diff --git a/doc/unlang/update.adoc b/doc/unlang/update.adoc index 647e37935c5..32780c04637 100644 --- a/doc/unlang/update.adoc +++ b/doc/unlang/update.adoc @@ -1,4 +1,3 @@ - == The update Statement .Syntax @@ -52,58 +51,7 @@ the __ qualifier is omitted, then each entry inside of the `update` section *must* be prefixed with a list name. For example, `&request:User-Name ...` -Allow values for the __ field are: - -`request`:: Attributes in the incoming request packet. - -`reply`:: Attributes in the outgoing reply packet. - -`control`:: Attributes in the internal "control" list that is -associated with the request. -+ -The `control` attributes are used to manage how the request is -processed. These attributes are never sent in any packet. - -`session-state`:: Attributes which are maintained across multi-packet -exchanges. - -There must be a colon `:` after the list name and before the attribute name. -This syntax helps the server to distinguish between list names and attribute -names. - -NOTE: In version 3, there were additional lists such as `proxy`, -`proxy-reply`, and `coa`. These lists have been removed in version 4. -See the link:subrequest.adoc[subrequest] statement for how to create -child requests. - -With the exception of `session-state`, all of the above lists are -ephemeral. That is, they exist for one packet exchange, and only one -packet exchange. When a reply is sent for a request, the above lists -and all attributes are deleted. There is no way to reference an -attribute from a previous packet. We recommend using a database to -track complex state. - -In some cases, requests are associated a multi-packet exchange. For -those situations, the `session-state` list is automatically saved when -a reply is sent, and it is automatically restored when the next packet -in sequence comes in. Once the packet exchange has been finished, the -`session-state` list is deleted. - -In some cases, there is a parent-child relationship between requests. -In those situations, it is possible for the policy rules in the child -to refer to attributes in the parent. This reference can be made by -prefixing the __ name with the `parent` qualifier. The key word -`outer` is also a synonym for `parent`. If there are multiple -parent-child relationships, the `parent` qualifier can be repeated. - -There is, however, no way for the parent to refer to the child. When -the child is running, the parent is suspended. Once the child -finishes, it is deleted, and is no longer accessible to the parent. - -.Examples -* `&parent.request:User-Name` -* `&parent.reply:Reply-Message` -* `&parent.parent.session-state:Filter-Id` +Please see the link:list.adoc[list] page for valid list names. === Server Attributes @@ -133,6 +81,8 @@ name is already present in that list, its value is replaced with the value of the current attribute. | += | Add the attribute to the tail of the list, even if attributes of the same name are already present in the list. +| -= | Remove all attributes from the list that match __. +| !* | Delete all occurances of the attribute, no matter what the value. |===== .Example @@ -151,43 +101,6 @@ update reply { } ---- -=== Filtering Operators - -The following operators may also be used in addition to the ones -listed above. These operators use the __ and -__ fields to enforce limits on all attributes in the given -__, and to edit attributes which have a matching -__ name. All other attributes are ignored. - -.Filtering Operators -[options="header] -[cols="10%,90%"] -|===== -| Operator | Description -| -= | Remove all attributes from the list that match __. -| == | Keep only the attributes in the list that match __ -| < | Keep only the attributes in the list that have values less than __. -| \<= | Keep only the attributes in the list that have values less than or equal to __. -| > | Keep only the attributes in the list that have values greater than __. -| >= | Keep only the attributes in the list that have values greater than or equal to __. -| !* | Delete all occurances of the attribute, no matter what the value. -|===== - -The `==` operator is very different from the `=` operator listed -above. The `=` operator is used to add new attributes to the list, -while the `==` operator removes all attributes that do not match the -given value. - -For IP addresses, the operators `>`, `>=`, `<`, and `<=` check for -membership in a network. The __ field should then be a IP -network, given in `address/mask` format. - -IMPORTANT: In version 3, some filtering operators would _create_ the -attribute if it did not exist in the destination list. In version 4, -this functionality has been removed. Instead, version 4 only performs -_filtering_ of the attributes. That is, the filtering operators will -modify or delete attributes, but they will never create an attribute. - === Values The __ field is the value which is assigned to the diff --git a/doc/upgrade/README.adoc b/doc/upgrade/README.adoc index 3281f23a668..e2d3286eeaf 100644 --- a/doc/upgrade/README.adoc +++ b/doc/upgrade/README.adoc @@ -393,15 +393,24 @@ still parse (but not generate) attributes of the form in version 4. The server would never produce such names, and enabling them made attribute parsing significantly more complex. -== Update sections and Filtering +== Update sections -The filtering operators in v4 have been modified. They no longer _set_ -the attribute to a value. Instead, they only _filter_ the attribute -list, and delete any attributes which do not match. The filtering -operators do not _create_ any attribute. +The filtering operators `<`, `>`, `<=`, `>=`, etc. have been moved to +the `filter` keyword. These operators will still be accepted in an +`update` section, but using them will generate a warning. + +== Filter Sections + +The `filter` section is now used to filter attributes. + +The filtering operators have been modified. They no longer _set_ the +attribute to a value. Instead, they only _filter_ the attribute list, +and delete any attributes which do not match. The filtering operators +do not _create_ any attribute. In order to achieve the same functionality as v3, you will need to set -the value of the attribute, in addition to filtering it. +the value of the attribute via `=` in an `update` statement, and then +filter it. == load-balance and redundant-load-balance sections diff --git a/src/lib/unlang/compile.c b/src/lib/unlang/compile.c index 8f0e3055dd3..c75fe4d2acd 100644 --- a/src/lib/unlang/compile.c +++ b/src/lib/unlang/compile.c @@ -1253,11 +1253,17 @@ int unlang_fixup_update(vp_map_t *map, UNUSED void *ctx) /* * Depending on the attribute type, some operators are disallowed. */ - if (tmpl_is_attr(map->lhs) && (!fr_assignment_op[map->op] && !fr_equality_op[map->op])) { - cf_log_err(map->ci, "Invalid operator \"%s\" in update section. " - "Only assignment or filter operators are allowed", - fr_int2str(fr_tokens_table, map->op, "")); - return -1; + if (tmpl_is_attr(map->lhs)) { + if (!fr_assignment_op[map->op] && !fr_equality_op[map->op]) { + cf_log_err(map->ci, "Invalid operator \"%s\" in update section. " + "Only assignment or filter operators are allowed", + fr_int2str(fr_tokens_table, map->op, "")); + return -1; + } + + if (fr_equality_op[map->op]) { + cf_log_warn(cp, "Please use the 'filter' keyword for attribute filtering"); + } } if (tmpl_is_list(map->lhs)) { @@ -1368,6 +1374,136 @@ int unlang_fixup_update(vp_map_t *map, UNUSED void *ctx) } +/** Validate and fixup a map that's part of a filter section. + * + * @param map to validate. + * @param ctx data to pass to fixup function (currently unused). + * @return + * - 0 if valid. + * - -1 not valid. + */ +static int unlang_fixup_filter(vp_map_t *map, UNUSED void *ctx) +{ + CONF_PAIR *cp = cf_item_to_pair(map->ci); + + /* + * Anal-retentive checks. + */ + if (DEBUG_ENABLED3) { + if (tmpl_is_attr(map->lhs) && (map->lhs->name[0] != '&')) { + cf_log_warn(cp, "Please change attribute reference to '&%s %s ...'", + map->lhs->name, fr_int2str(fr_tokens_table, map->op, "")); + } + + if (tmpl_is_attr(map->rhs) && (map->rhs->name[0] != '&')) { + cf_log_warn(cp, "Please change attribute reference to '... %s &%s'", + fr_int2str(fr_tokens_table, map->op, ""), map->rhs->name); + } + } + + /* + * We only allow attributes on the LHS. + */ + if (map->lhs->type != TMPL_TYPE_ATTR) { + cf_log_err(cp, "Filter sections can only operate on attributes"); + return -1; + } + + if (map->rhs->type == TMPL_TYPE_LIST) { + cf_log_err(map->ci, "Cannot filter an attribute using a list."); + return -1; + } + + /* + * Fixup LHS attribute references to change NUM_ANY to NUM_ALL. + */ + if (map->lhs->tmpl_num == NUM_ANY) map->lhs->tmpl_num = NUM_ALL; + + /* + * Fixup RHS attribute references to change NUM_ANY to NUM_ALL. + */ + if ((map->rhs->type == TMPL_TYPE_ATTR) && + (map->rhs->tmpl_num == NUM_ANY)) { + map->rhs->tmpl_num = NUM_ALL; + } + + /* + * Values used by unary operators should be literal ANY + * + * We then free the template and alloc a NULL one instead. + */ + if (map->op == T_OP_CMP_FALSE) { + if (!tmpl_is_unparsed(map->rhs) || (strcmp(map->rhs->name, "ANY") != 0)) { + WARN("%s[%d] Wildcard deletion MUST use '!* ANY'", + cf_filename(cp), cf_lineno(cp)); + } + + TALLOC_FREE(map->rhs); + + map->rhs = tmpl_alloc(map, TMPL_TYPE_NULL, NULL, 0, T_INVALID); + } + + /* + * Lots of sanity checks for insane people... + */ + + /* + * Filtering only allows for filtering operators. + */ + if (tmpl_is_attr(map->lhs) && !fr_equality_op[map->op]) { + cf_log_err(map->ci, "Invalid operator \"%s\" in update section. " + "Only assignment or filter operators are allowed", + fr_int2str(fr_tokens_table, map->op, "")); + return -1; + } + + /* + * If the map has a unary operator there's no further + * processing we need to, as RHS is unused. + */ + if (map->op == T_OP_CMP_FALSE) return 0; + + /* + * If LHS is an attribute, and RHS is a literal, we can + * preparse the information into a TMPL_TYPE_DATA. + * + * Unless it's a unary operator in which case we + * ignore map->rhs. + */ + if (tmpl_is_attr(map->lhs) && tmpl_is_unparsed(map->rhs)) { + /* + * It's a literal string, just copy it. + * Don't escape anything. + */ + if (tmpl_cast_in_place(map->rhs, map->lhs->tmpl_da->type, map->lhs->tmpl_da) < 0) { + cf_log_perr(map->ci, "Cannot convert RHS value (%s) to LHS attribute type (%s)", + fr_int2str(fr_value_box_type_table, FR_TYPE_STRING, ""), + fr_int2str(fr_value_box_type_table, map->lhs->tmpl_da->type, "")); + return -1; + } + + /* + * Fixup LHS da if it doesn't match the type + * of the RHS. + */ + if (map->lhs->tmpl_da->type != map->rhs->tmpl_value_type) { + fr_dict_attr_t const *da; + + da = fr_dict_attr_by_type(map->lhs->tmpl_da, map->rhs->tmpl_value_type); + if (!da) { + fr_strerror_printf("Cannot find %s variant of attribute \"%s\"", + fr_int2str(fr_value_box_type_table, map->rhs->tmpl_value_type, + ""), map->lhs->tmpl_da->name); + return -1; + } + map->lhs->tmpl_da = da; + } + } /* else we can't precompile the data */ + + return 0; +} + + static unlang_group_t *group_allocate(unlang_t *parent, CONF_SECTION *cs, unlang_group_type_t group_type, unlang_type_t mod_type) { @@ -1647,6 +1783,67 @@ static unlang_t *compile_update(unlang_t *parent, unlang_compile_t *unlang_ctx, return c; } +static unlang_t *compile_filter(unlang_t *parent, unlang_compile_t *unlang_ctx, + CONF_SECTION *cs, unlang_group_type_t group_type, + UNUSED unlang_group_type_t parentgroup_type, UNUSED unlang_type_t mod_type) +{ + int rcode; + unlang_group_t *g; + unlang_t *c; + char const *name2 = cf_section_name2(cs); + + vp_map_t *head; + + vp_tmpl_rules_t parse_rules; + + /* + * We allow unknown attributes here. + */ + parse_rules = *(unlang_ctx->rules); + parse_rules.allow_unknown = true; + + /* + * This looks at cs->name2 to determine which list to update + */ + rcode = map_afrom_cs(cs, &head, cs, &parse_rules, &parse_rules, unlang_fixup_filter, NULL, 128); + if (rcode < 0) return NULL; /* message already printed */ + if (!head) { + cf_log_err(cs, "'update' sections cannot be empty"); + return NULL; + } + + g = group_allocate(parent, cs, group_type, UNLANG_TYPE_FILTER); + if (!g) return NULL; + + c = unlang_group_to_generic(g); + + if (name2) { + c->name = name2; + c->debug_name = talloc_typed_asprintf(c, "filter %s", name2); + } else { + c->name = unlang_ops[c->type].name; + c->debug_name = unlang_ops[c->type].name; + } + + (void) compile_action_defaults(c, unlang_ctx, UNLANG_GROUP_TYPE_SIMPLE); + + g->map = talloc_steal(g, head); + +#ifdef WITH_CONF_WRITE +// cf_data_add(cs, CF_DATA_TYPE_FILTER, "filter", g->map, NULL); /* for output normalization */ +#endif + + /* + * The fixups here occur whether or not it's UPDATE or FILTER + */ + if (!pass2_fixup_update(g, unlang_ctx->rules)) { + talloc_free(g); + return NULL; + } + + return c; +} + /* * Compile action && rcode for later use. */ @@ -2540,10 +2737,14 @@ static int all_children_are_modules(CONF_SECTION *cs, char const *name) CONF_SECTION *subcs = cf_item_to_section(ci); char const *name1 = cf_section_name1(subcs); + /* + * @todo - put this into a list somewhere + */ if ((strcmp(name1, "if") == 0) || (strcmp(name1, "else") == 0) || (strcmp(name1, "elsif") == 0) || (strcmp(name1, "update") == 0) || + (strcmp(name1, "filter") == 0) || (strcmp(name1, "switch") == 0) || (strcmp(name1, "case") == 0)) { cf_log_err(ci, "%s sections cannot contain a \"%s\" statement", @@ -3149,6 +3350,7 @@ static modcall_compile_t compile_table[] = { { "if", compile_if, UNLANG_GROUP_TYPE_SIMPLE, UNLANG_TYPE_IF, ALLOW_EMPTY_GROUP }, { "elsif", compile_elsif, UNLANG_GROUP_TYPE_SIMPLE, UNLANG_TYPE_ELSIF, ALLOW_EMPTY_GROUP }, { "else", compile_else, UNLANG_GROUP_TYPE_SIMPLE, UNLANG_TYPE_ELSE, REQUIRE_CHILDREN }, + { "filter", compile_filter, UNLANG_GROUP_TYPE_SIMPLE, UNLANG_TYPE_FILTER, REQUIRE_CHILDREN }, { "update", compile_update, UNLANG_GROUP_TYPE_SIMPLE, UNLANG_TYPE_UPDATE, REQUIRE_CHILDREN }, { "map", compile_map, UNLANG_GROUP_TYPE_SIMPLE, UNLANG_TYPE_MAP, REQUIRE_CHILDREN }, { "switch", compile_switch, UNLANG_GROUP_TYPE_SIMPLE, UNLANG_TYPE_SWITCH, REQUIRE_CHILDREN }, diff --git a/src/lib/unlang/map.c b/src/lib/unlang/map.c index 2e4ee3cbf99..d954ec6b972 100644 --- a/src/lib/unlang/map.c +++ b/src/lib/unlang/map.c @@ -302,6 +302,16 @@ static unlang_action_t unlang_map(REQUEST *request, rlm_rcode_t *presult, int *p void unlang_map_init(void) { + /* + * For now, FILTER and UPDATE use the same processor. + */ + unlang_register(UNLANG_TYPE_FILTER, + &(unlang_op_t){ + .name = "filter", + .func = unlang_update, + .debug_braces = true + }); + unlang_register(UNLANG_TYPE_UPDATE, &(unlang_op_t){ .name = "update", diff --git a/src/lib/unlang/unlang_priv.h b/src/lib/unlang/unlang_priv.h index a7fe96d9541..3fa9918708c 100644 --- a/src/lib/unlang/unlang_priv.h +++ b/src/lib/unlang/unlang_priv.h @@ -65,6 +65,7 @@ typedef enum { UNLANG_TYPE_IF, //!< Condition. UNLANG_TYPE_ELSE, //!< !Condition. UNLANG_TYPE_ELSIF, //!< !Condition && Condition. + UNLANG_TYPE_FILTER, //!< Filter block. UNLANG_TYPE_UPDATE, //!< Update block. UNLANG_TYPE_SWITCH, //!< Switch section. UNLANG_TYPE_CASE, //!< Case section (within a #UNLANG_TYPE_SWITCH). @@ -155,7 +156,7 @@ typedef struct { union { struct { - vp_map_t *map; //!< #UNLANG_TYPE_UPDATE, #UNLANG_TYPE_MAP. + vp_map_t *map; //!< #UNLANG_TYPE_FILTER, #UNLANG_TYPE_UPDATE, #UNLANG_TYPE_MAP, map_proc_inst_t *proc_inst; //!< Instantiation data for #UNLANG_TYPE_MAP. }; struct { diff --git a/src/tests/keywords/filter b/src/tests/keywords/filter new file mode 100644 index 00000000000..17c04e74f86 --- /dev/null +++ b/src/tests/keywords/filter @@ -0,0 +1,32 @@ +# +# PRE: update +# +update request { + &Session-Timeout := 100 +} + +filter request { + &Session-Timeout < 50 +} + +if (&Session-Timeout) { + fail +} + +update request { + &Session-Timeout := 100 +} + +filter request { + &Session-Timeout < 200 +} + +if (!&Session-Timeout) { + fail +} + +if (&Session-Timeout != 100) { + fail +} + +success