lib-settings, config: Fix and simplify override order handling

The order is now based on the number of named list filter elements in the
override path vs binary config's event filter.

This also happens to fix at least some issues where @group include was
wrongly overriding userdb fields.

diff --git a/src/config/config-dump-full.c b/src/config/config-dump-full.c
index b87eeab8a62461848a57d2dbda9a9e6481990b0e..a2138588882275c74f641da387ec58bb229c49b6 100644 (file)
--- a/src/config/config-dump-full.c
+++ b/src/config/config-dump-full.c
@@ -47,6 +47,7 @@
    Repeat for "event filter strings count":
      <NUL-terminated string: event filter string>
      <NUL-terminated string: override event filter string>
+     <32bit: number of named list filter elements>
 
    Repeat until "settings full size" is reached:
      <64bit: settings block size>
@@ -442,6 +443,17 @@ config_dump_full_append_filter(string_t *str,
                str_truncate(str, str_len(str) - 5);
 }
 
+static uint32_t
+config_filter_get_name_list_counts(const struct config_filter *filter)
+{
+       uint32_t count = 0;
+       for (; filter != NULL; filter = filter->parent) {
+               if (filter->filter_name_array)
+                       count++;
+       }
+       return count;
+}
+
 static void
 config_dump_full_write_filters(struct ostream *output,
                               struct config_parsed *config)
@@ -456,8 +468,12 @@ config_dump_full_write_filters(struct ostream *output,
        o_stream_nsend(output, &filter_count_32, sizeof(filter_count_32));
 
        /* the first filter is the global empty filter */
+       uint32_t named_list_filter_count = 0;
        o_stream_nsend(output, "", 1);
        o_stream_nsend(output, "", 1);
+       o_stream_nsend(output, &named_list_filter_count,
+                      sizeof(named_list_filter_count));
+
        string_t *str = str_new(default_pool, 128);
        for (i = 1; i < filter_count; i++) T_BEGIN {
                str_truncate(str, 0);
@@ -469,6 +485,11 @@ config_dump_full_write_filters(struct ostream *output,
                config_dump_full_append_filter(str, &filters[i]->filter, FALSE);
                str_append_c(str, '\0');
                o_stream_nsend(output, str_data(str), str_len(str));
+
+               named_list_filter_count =
+                       config_filter_get_name_list_counts(&filters[i]->filter);
+               o_stream_nsend(output, &named_list_filter_count,
+                              sizeof(named_list_filter_count));
        } T_END;
        str_free(&str);
 }

diff --git a/src/lib-master/test-master-service-settings.c b/src/lib-master/test-master-service-settings.c
index ff2f11fee20631a2397122a453b58db728fdcc3b..8d40381a9133b76564bd239843504f24da2d4a4a 100644 (file)
--- a/src/lib-master/test-master-service-settings.c
+++ b/src/lib-master/test-master-service-settings.c
@@ -97,7 +97,7 @@ static const struct {
        /* full file size is 7 bytes, which makes the first block size
           truncated, since it needs 8 bytes */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x22") // full size
+              NUM64("\x26") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -107,11 +107,12 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               "\x00\x00\x00\x00\x00\x00\x00"), // block size
          "Area too small when reading size of 'block size'" },
        /* first block size is 0, which is too small */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x23") // full size
+              NUM64("\x27") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -121,11 +122,12 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x00")), // block size
          "'block name' points outside area" },
        /* first block size is 1, but full file size is too small */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x23") // full size
+              NUM64("\x27") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -135,11 +137,12 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x01")), // block size
          "'block size' points outside are" },
        /* block name is not NUL-terminated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x25") // full size
+              NUM64("\x29") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -149,6 +152,7 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x01") // block size
               "N"
               "\x00"), // trailing garbage so we can have NUL
@@ -156,7 +160,7 @@ static const struct {
 
        /* settings count is truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x28") // full size
+              NUM64("\x2C") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -166,6 +170,7 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x05") // block size
               "N\x00" // block name
               "\x00\x00\x00"),
@@ -173,7 +178,7 @@ static const struct {
 
        /* settings keys are truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x29") // full size
+              NUM64("\x2D") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -183,6 +188,7 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x06") // block size
               "N\x00" // block name
               NUM32("\x01")), // settings count
@@ -190,7 +196,7 @@ static const struct {
 
        /* filter count is truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x2E") // full size
+              NUM64("\x32") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -200,6 +206,7 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x0B") // block size
               "N\x00" // block name
               NUM32("\x01") // settings count
@@ -209,7 +216,7 @@ static const struct {
 
        /* filter settings size is truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x36") // full size
+              NUM64("\x3A") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -219,6 +226,7 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x12") // block size
               "N\x00" // block name
               NUM32("\x01") // settings count
@@ -229,7 +237,7 @@ static const struct {
 
        /* filter settings is truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x37") // full size
+              NUM64("\x3B") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -239,6 +247,7 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x14") // block size
               "N\x00" // block name
               NUM32("\x01") // settings count
@@ -248,7 +257,7 @@ static const struct {
          "'filter settings size' points outside area" },
        /* filter error is missing */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x44") // full size
+              NUM64("\x48") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -258,6 +267,7 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x21") // block size
               "N\x00" // block name
               NUM32("\x01") // settings count
@@ -270,7 +280,7 @@ static const struct {
          "'filter error string' points outside area" },
        /* filter error is not NUL-terminated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x52") // full size
+              NUM64("\x56") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -280,6 +290,7 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x2F") // block size
               "master_service\x00" // block name
               NUM32("\x01") // settings count
@@ -293,7 +304,7 @@ static const struct {
          "'filter error string' points outside area" },
        /* include group count is truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x55") // full size
+              NUM64("\x59") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -303,6 +314,7 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x32") // block size
               "master_service\x00" // block name
               NUM32("\x01") // settings count
@@ -317,7 +329,7 @@ static const struct {
          "Area too small when reading uint of 'include group count'" },
        /* include group count is too large */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x56") // full size
+              NUM64("\x5A") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -327,6 +339,7 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x33") // block size
               "master_service\x00" // block name
               NUM32("\x01") // settings count
@@ -341,7 +354,7 @@ static const struct {
          "'group label string' points outside area" },
        /* group label not NUL-terminated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x57") // full size
+              NUM64("\x5B") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -351,6 +364,7 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x34") // block size
               "master_service\x00" // block name
               NUM32("\x01") // settings count
@@ -366,7 +380,7 @@ static const struct {
          "'group label string' points outside area" },
        /* group name not NUL-terminated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x59") // full size
+              NUM64("\x5D") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -376,6 +390,7 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x36") // block size
               "master_service\x00" // block name
               NUM32("\x01") // settings count
@@ -392,7 +407,7 @@ static const struct {
          "'group name string' points outside area" },
        /* invalid filter string */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x4B") // full size
+              NUM64("\x4F") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -402,6 +417,7 @@ static const struct {
               NUM32("\x01") // event filter count
               "F\x00" // event filter[0]
               "F\x00" // override event filter[0]
+              NUM32("\x00") // number of named list filter elements
               NUM64("\x26") // block size
               "N\x00" // block name
               NUM32("\x01") // settings count
@@ -418,7 +434,7 @@ static const struct {
 
        /* Duplicate block name */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x54") // full size
+              NUM64("\x5C") // full size
               NUM32("\x00") // cache path count
               NUM32("\x0D") // all keys size
               "\x00" // 32bit padding
@@ -428,7 +444,8 @@ static const struct {
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
-              NUM64("\x27") // block size
+              NUM32("\x00") // number of named list filter elements
+              NUM64("\x2B") // block size
               "N\x00" // block name
               NUM32("\x01") // settings count
               "K\x00" // setting[0] key
@@ -436,7 +453,7 @@ static const struct {
               NUM64("\x05") // filter settings size
               "\x00" // filter error string
               NUM32("\x00") // include group count
-              "\x00" // 64bit padding
+              "\x00\x00\x00\x00\x00" // 64bit padding
               NUM64("\x00") // filter[0] settings offset
               NUM32("\x00") // filter[0] event filter index
               "\x00" // safety NUL

diff --git a/src/lib-settings/settings.c b/src/lib-settings/settings.c
index 28490a15bb087e638fbbae0ef7d8ee4523d9838e..af3a73738b2d6f0369016d2266f2807f7e335da6 100644 (file)
--- a/src/lib-settings/settings.c
+++ b/src/lib-settings/settings.c
@@ -52,8 +52,6 @@ struct settings_override {
        unsigned int filter_element_count;
        /* key += value is used, i.e. append this value to existing value */
        bool append;
-       /* Always apply this override, regardless of any filters. */
-       bool always_match;
        bool array_default_added;
        /* Original key for the overridden setting, e.g.
           namespace/inbox/mailbox/Sent/mail_attribute/dict_driver */
@@ -108,6 +106,8 @@ struct settings_mmap_event_filter {
        struct event_filter *filter;
        struct event_filter *override_filter;
        bool is_group;
+
+       uint32_t named_list_filter_count;
 };
 
 struct settings_mmap {
@@ -196,7 +196,7 @@ static struct event_filter event_filter_match_never, event_filter_match_always;
 
 static int
 settings_instance_override(struct settings_apply_ctx *ctx,
-                          struct event_filter *event_filter,
+                          struct settings_mmap_event_filter *set_filter,
                           bool *defaults, const char **error_r);
 
 static bool settings_local_name_cmp(const char *value, const char *wanted_value)
@@ -519,6 +519,14 @@ settings_read_filters(struct settings_mmap *mmap, const char *service_name,
                                            "filter string", &filter_string,
                                            error_r) < 0)
                        return -1;
+               if (i % 2 != 0) {
+                       if (settings_block_read_uint32(mmap, offset, mmap->mmap_size,
+                                       "named list filter element count",
+                                       &set_filter->named_list_filter_count,
+                                       error_r) < 0)
+                               return -1;
+               }
+
                if (filter_string[0] == '\0') {
                        *filter_dest = EVENT_FILTER_MATCH_ALWAYS;
                        continue;
@@ -1201,7 +1209,7 @@ settings_mmap_apply_filter(struct settings_apply_ctx *ctx,
           override is specifically using the filter name
           as prefix. */
        bool defaults = FALSE;
-       int ret = settings_instance_override(ctx, set_filter->override_filter,
+       int ret = settings_instance_override(ctx, set_filter,
                                             &defaults, error_r);
        if (ret < 0)
                return -1;
@@ -1214,9 +1222,8 @@ settings_mmap_apply_filter(struct settings_apply_ctx *ctx,
                                     error_r) < 0)
                return -1;
        if (defaults) {
-               ret = settings_instance_override(ctx,
-                               set_filter->override_filter,
-                               &defaults, error_r);
+               ret = settings_instance_override(ctx, set_filter,
+                                                &defaults, error_r);
                if (ret < 0)
                        return -1;
                if (ret > 0)
@@ -1716,6 +1723,14 @@ settings_apply_override_cmp(const struct settings_apply_override *set1,
        return set2->order - set1->order;
 }
 
+static int
+settings_override_cmp_filter_order(const struct settings_override *set,
+                                  struct settings_mmap_event_filter *set_filter)
+{
+       return (int)set->filter_array_element_count -
+               (int)set_filter->named_list_filter_count;
+}
+
 static bool
 settings_mmap_registered_lookup_key(const char *key, enum setting_type *type_r)
 {
@@ -1921,8 +1936,9 @@ settings_override_filter_match(struct settings_apply_ctx *ctx,
                                                       set, error_r);
                if (ret <= 0)
                        return ret;
+               /* Sort this as the first */
+               set->filter_array_element_count = INT_MAX;
                set->filter = EVENT_FILTER_MATCH_ALWAYS;
-               set->always_match = TRUE;
                return setting_override_match_info(ctx, set) ? 1 : 0;
        }
 
@@ -2323,14 +2339,11 @@ settings_include_group_add_or_update(ARRAY_TYPE(settings_group) *include_groups,
 
 static int
 settings_instance_override(struct settings_apply_ctx *ctx,
-                          struct event_filter *event_filter,
+                          struct settings_mmap_event_filter *set_filter,
                           bool *defaults, const char **error_r)
 {
        struct settings_apply_override *override;
        struct settings_override *set;
-       const struct failure_context failure_ctx = {
-               .type = LOG_TYPE_DEBUG
-       };
        enum setting_apply_flags apply_flags = SETTING_APPLY_FLAG_OVERRIDE |
                ((ctx->flags & SETTINGS_GET_FLAG_NO_EXPAND) == 0 ? 0 :
                 SETTING_APPLY_FLAG_NO_EXPAND);
@@ -2348,12 +2361,9 @@ settings_instance_override(struct settings_apply_ctx *ctx,
                /* If we're being called while applying filters, only apply
                   the overrides that have a matching filter. This preserves
                   the expected order in which settings are applied. */
-               if (!set->always_match &&
-                   event_filter != EVENT_FILTER_MATCH_ALWAYS &&
-                   (set->filter_event == NULL ||
-                    !event_filter_match(event_filter, set->filter_event,
-                                        &failure_ctx)))
-                       continue;
+               if (set_filter->filter != EVENT_FILTER_MATCH_ALWAYS &&
+                   settings_override_cmp_filter_order(set, set_filter) < 0)
+                       break;
 
                if (set->type == SETTINGS_OVERRIDE_TYPE_DEFAULT) {
                        seen_defaults = TRUE;
@@ -2559,7 +2569,9 @@ settings_instance_get(struct settings_apply_ctx *ctx,
        } else {
                /* No configuration file - apply all overrides */
                bool defaults = TRUE;
-               ret = settings_instance_override(ctx, EVENT_FILTER_MATCH_ALWAYS,
+               struct settings_mmap_event_filter set_filter = {
+               };
+               ret = settings_instance_override(ctx, &set_filter,
                                                 &defaults, error_r);
        }
        if (ret > 0)