config, lib-settings: Write all setting keys into hash table in binary config

diff --git a/src/config/config-dump-full.c b/src/config/config-dump-full.c
index 80af484f8ca85d3c739bb7612d6b86af34ece65a..b87eeab8a62461848a57d2dbda9a9e6481990b0e 100644 (file)
--- a/src/config/config-dump-full.c
+++ b/src/config/config-dump-full.c
@@ -2,8 +2,10 @@
 
 #include "lib.h"
 #include "array.h"
+#include "hash.h"
 #include "str.h"
 #include "strescape.h"
+#include "primes.h"
 #include "wildcard-match.h"
 #include "safe-mkstemp.h"
 #include "ostream.h"
@@ -12,6 +14,7 @@
 #include "config-parser.h"
 #include "config-request.h"
 #include "config-filter.h"
+#include "all-settings.h"
 #include "config-dump-full.h"
 
 #include <stdio.h>
@@ -37,6 +40,9 @@
      <32bit: ctime UNIX timestamp>
      <32bit: ctime nsecs>
 
+   <32bit: all setting keys size>
+   <all setting keys - see below>
+
    <32bit: event filter strings count>
    Repeat for "event filter strings count":
      <NUL-terminated string: event filter string>
@@ -84,6 +90,30 @@
    file. This automatically causes the more specific filters to be written
    after the less specific ones.
 
+   All Setting Keys
+   ----------------
+
+   This hash table contains all the setting names and their details. It is
+   needed for handling setting overrides in an efficient way. It is guaranteed
+   that there are no collisions in the written hash table.
+
+   The <all setting keys> contents are:
+
+   [base offset for nodes]
+   <0..3 bytes to pad to 32bit offset>
+   <32bit: hash table key prefix>
+   <32bit: hash table nodes count>
+     <32bit: setting node relative offset>
+   <32bit: setting block names count>
+     <32bit: setting block name relative offset>
+
+   The <setting node> contents are:
+
+   <NUL-terminated string: setting name>
+   <32bit: enum setting_type>
+   <32bit: setting blocks count>
+     <32bit: setting block name index>
+
    Groups
    ------
 
@@ -196,6 +226,157 @@ config_dump_full_write_cache_paths(struct ostream *output,
                                                                    main_path->path);
 }
 
+static bool
+config_dump_keys_hash_try(struct config_parsed *config,
+                         uint32_t key_prefix, unsigned int hash_table_size)
+{
+       const HASH_TABLE_TYPE(config_key) all_keys =
+               config_parsed_get_all_keys(config);
+       const char *name;
+       struct config_parser_key *config_key;
+       bool ret = TRUE;
+
+       bool *table = i_new(bool, hash_table_size);
+       struct hash_iterate_context *iter = hash_table_iterate_init(all_keys);
+       while (hash_table_iterate(iter, all_keys, &name, &config_key)) {
+               uint32_t key_hash = (key_prefix ^ str_hash(name)) %
+                       hash_table_size;
+               if (table[key_hash]) {
+                       ret = FALSE;
+                       break;
+               }
+               table[key_hash] = TRUE;
+       }
+       hash_table_iterate_deinit(&iter);
+       i_free(table);
+       return ret;
+}
+
+static unsigned int
+config_parser_key_list_count(struct config_parser_key *config_key)
+{
+       unsigned int i;
+       for (i = 0; config_key != NULL; i++)
+               config_key = config_key->next;
+       return i;
+}
+
+static void
+config_dump_keys_hash(struct config_parsed *config, buffer_t *buf,
+                     uint32_t key_prefix, uint32_t hash_table_size)
+{
+       const HASH_TABLE_TYPE(config_key) all_keys =
+               config_parsed_get_all_keys(config);
+       const char *name;
+       struct config_parser_key *config_key;
+
+       /* hash table key prefix */
+       buffer_append(buf, &key_prefix, sizeof(key_prefix));
+       /* hash table nodes count */
+       buffer_append(buf, &hash_table_size, sizeof(hash_table_size));
+       /* reserve space for the hash table */
+       size_t hash_table_offset = buf->used;
+       buffer_append_zero(buf, sizeof(uint32_t) * hash_table_size);
+
+       /* setting block names */
+       uint32_t count;
+       for (count = 0; all_infos[count] != NULL; count++) ;
+       buffer_append(buf, &count, sizeof(count));
+       size_t blocks_offset = buf->used;
+       buffer_append_zero(buf, sizeof(uint32_t) * count);
+       for (uint32_t i = 0; i < count; i++) {
+               uint32_t offset = buf->used;
+               buffer_write(buf, blocks_offset + i * sizeof(uint32_t),
+                            &offset, sizeof(offset));
+               buffer_append(buf, all_infos[i]->name,
+                             strlen(all_infos[i]->name) + 1);
+       }
+
+       /* setting nodes */
+       uint32_t *hash_table = i_new(uint32_t, hash_table_size);
+       struct hash_iterate_context *iter = hash_table_iterate_init(all_keys);
+       while (hash_table_iterate(iter, all_keys, &name, &config_key)) {
+               uint32_t key_hash = (key_prefix ^ str_hash(name)) %
+                       hash_table_size;
+               i_assert(hash_table[key_hash] == 0);
+
+               hash_table[key_hash] = buf->used;
+               buffer_append(buf, name, strlen(name) + 1);
+
+               const struct setting_define *def =
+                       &all_infos[config_key->info_idx]->defines[config_key->define_idx];
+               uint32_t num32 = def->type;
+               buffer_append(buf, &num32, sizeof(num32));
+
+               num32 = config_parser_key_list_count(config_key);
+               buffer_append(buf, &num32, sizeof(num32));
+
+               do {
+                       num32 = config_key->info_idx;
+                       buffer_append(buf, &num32, sizeof(num32));
+                       config_key = config_key->next;
+               } while (config_key != NULL);
+       }
+       hash_table_iterate_deinit(&iter);
+
+       buffer_write(buf, hash_table_offset, hash_table,
+                    sizeof(uint32_t) * hash_table_size);
+       i_free(hash_table);
+}
+
+static void
+config_dump_keys_hash_table(struct config_parsed *config, buffer_t *buf)
+{
+       const HASH_TABLE_TYPE(config_key) all_keys =
+               config_parsed_get_all_keys(config);
+       unsigned int i, hash_table_size = hash_table_count(all_keys);
+
+       /* Find a hash table where no settings keys have collisions. We can
+          spend more time here so later settings lookups are more efficient by
+          not having to handle key collisions. */
+       for (;;) {
+               /* Try a few times to fit the hash table into a smaller node
+                  count by using different key prefixes. Use predictable key
+                  prefixes, so the hash table sizes are also predictable (and
+                  easier to debug) across different runs. */
+               for (i = 0; i < 10; i++) {
+                       uint32_t key_prefix = i;
+                       if (config_dump_keys_hash_try(config, key_prefix,
+                                                     hash_table_size)) {
+                               config_dump_keys_hash(config, buf,
+                                       key_prefix, hash_table_size);
+                               return;
+                       }
+               }
+               /* Continue with a larger hash table size until we succeed.
+                  In theory this could cause huge hash table sizes (or even
+                  out of memory), but practically that shouldn't happen. */
+               hash_table_size = primes_closest(hash_table_size + 1);
+       }
+}
+
+static void
+config_dump_full_write_all_keys(struct ostream *output,
+                               struct config_parsed *config)
+{
+       buffer_t *buf = buffer_create_dynamic(default_pool, 10240);
+
+       /* 32bit padding */
+       if (output->offset % sizeof(uint32_t) != 0) {
+               buffer_append_zero(buf, sizeof(uint32_t) -
+                                  output->offset % sizeof(uint32_t));
+       }
+       config_dump_keys_hash_table(config, buf);
+
+       /* all setting keys size */
+       i_assert(buf->used <= UINT32_MAX);
+       uint32_t set_size = buf->used;
+
+       o_stream_nsend(output, &set_size, sizeof(set_size));
+       o_stream_nsend(output, buf->data, buf->used);
+       buffer_free(&buf);
+}
+
 static void
 config_dump_full_append_filter_query(string_t *str,
                                     const struct config_filter *filter,
@@ -762,6 +943,7 @@ int config_dump_full(struct config_parsed *config,
                                                   &cache_path);
                if (cache_path != NULL)
                        final_path = cache_path;
+               config_dump_full_write_all_keys(output, config);
                config_dump_full_write_filters(output, config);
        }
 

diff --git a/src/lib-master/test-master-service-settings.c b/src/lib-master/test-master-service-settings.c
index d3a614fd145b44d5c79f85e5a43fd4fc6567a9c3..ff2f11fee20631a2397122a453b58db728fdcc3b 100644 (file)
--- a/src/lib-master/test-master-service-settings.c
+++ b/src/lib-master/test-master-service-settings.c
@@ -44,25 +44,66 @@ static const struct {
               "\x00\x00\x00"), // cache path count
          "Area too small when reading uint of 'config paths count'" },
 
-       /* event filter count is truncated */
+       /* all keys size is truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
               NUM64("\x07") // full size
               NUM32("\x00") // cache path count
+              "\x00\x00\x00"), // all keys size
+         "Area too small when reading uint of 'all keys size'" },
+
+       /* all keys hash key prefix is truncated */
+       { DATA("DOVECOT-CONFIG\t1.0\n"
+              NUM64("\x0C") // full size
+              NUM32("\x00") // cache path count
+              NUM32("\x04") // all keys size
+              "\x00" // 32bit padding
+              "\x00\x00\x00"), // all keys hash key prefix
+         "Area too small when reading uint of 'all keys hash key prefix'" },
+
+         /* event all keys hash nodes count is truncated */
+       { DATA("DOVECOT-CONFIG\t1.0\n"
+              NUM64("\x10") // full size
+              NUM32("\x00") // cache path count
+              NUM32("\x08") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              "\x00\x00\x00"), // all keys hash nodes count
+         "Area too small when reading uint of 'all keys hash nodes count'" },
+
+       /* event filter count is truncated */
+       { DATA("DOVECOT-CONFIG\t1.0\n"
+              NUM64("\x18") // full size
+              NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               "\x00\x00\x00"), // event filter count
          "Area too small when reading uint of 'filters count'" },
 
        /* event filter strings are truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x08") // full size
+              NUM64("\x19") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01")), // event filter count
          "'filter string' points outside area" },
 
        /* full file size is 7 bytes, which makes the first block size
           truncated, since it needs 8 bytes */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x11") // full size
+              NUM64("\x22") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -70,8 +111,13 @@ static const struct {
          "Area too small when reading size of 'block size'" },
        /* first block size is 0, which is too small */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x12") // full size
+              NUM64("\x23") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -79,8 +125,13 @@ static const struct {
          "'block name' points outside area" },
        /* first block size is 1, but full file size is too small */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x12") // full size
+              NUM64("\x23") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -88,8 +139,13 @@ static const struct {
          "'block size' points outside are" },
        /* block name is not NUL-terminated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x14") // full size
+              NUM64("\x25") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -100,8 +156,13 @@ static const struct {
 
        /* settings count is truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x17") // full size
+              NUM64("\x28") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -112,8 +173,13 @@ static const struct {
 
        /* settings keys are truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x18") // full size
+              NUM64("\x29") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -124,8 +190,13 @@ static const struct {
 
        /* filter count is truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x1D") // full size
+              NUM64("\x2E") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -138,8 +209,13 @@ static const struct {
 
        /* filter settings size is truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x25") // full size
+              NUM64("\x36") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -153,8 +229,13 @@ static const struct {
 
        /* filter settings is truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x26") // full size
+              NUM64("\x37") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -167,8 +248,13 @@ static const struct {
          "'filter settings size' points outside area" },
        /* filter error is missing */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x33") // full size
+              NUM64("\x44") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -184,8 +270,13 @@ static const struct {
          "'filter error string' points outside area" },
        /* filter error is not NUL-terminated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x41") // full size
+              NUM64("\x52") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -202,8 +293,13 @@ static const struct {
          "'filter error string' points outside area" },
        /* include group count is truncated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x44") // full size
+              NUM64("\x55") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -221,8 +317,13 @@ static const struct {
          "Area too small when reading uint of 'include group count'" },
        /* include group count is too large */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x45") // full size
+              NUM64("\x56") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -240,8 +341,13 @@ static const struct {
          "'group label string' points outside area" },
        /* group label not NUL-terminated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x46") // full size
+              NUM64("\x57") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -260,8 +366,13 @@ static const struct {
          "'group label string' points outside area" },
        /* group name not NUL-terminated */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x48") // full size
+              NUM64("\x59") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
@@ -281,8 +392,13 @@ static const struct {
          "'group name string' points outside area" },
        /* invalid filter string */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x3A") // full size
+              NUM64("\x4B") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "F\x00" // event filter[0]
               "F\x00" // override event filter[0]
@@ -302,12 +418,17 @@ static const struct {
 
        /* Duplicate block name */
        { DATA("DOVECOT-CONFIG\t1.0\n"
-              NUM64("\x44") // full size
+              NUM64("\x54") // full size
               NUM32("\x00") // cache path count
+              NUM32("\x0D") // all keys size
+              "\x00" // 32bit padding
+              NUM32("\x00") // all keys hash key prefix
+              NUM32("\x00") // all keys hash nodes count
+              NUM32("\x00") // block names count
               NUM32("\x01") // event filter count
               "\x00" // event filter[0]
               "\x00" // override event filter[0]
-              NUM64("\x28") // block size
+              NUM64("\x27") // block size
               "N\x00" // block name
               NUM32("\x01") // settings count
               "K\x00" // setting[0] key
@@ -315,7 +436,7 @@ static const struct {
               NUM64("\x05") // filter settings size
               "\x00" // filter error string
               NUM32("\x00") // include group count
-              "\x00\x00" // 64bit padding
+              "\x00" // 64bit padding
               NUM64("\x00") // filter[0] settings offset
               NUM32("\x00") // filter[0] event filter index
               "\x00" // safety NUL

diff --git a/src/lib-settings/settings.c b/src/lib-settings/settings.c
index 3f75cea7d652c1699d91c058caa63bf29083b537..efeced1d54ea335dd556ecde3638a49736c53301 100644 (file)
--- a/src/lib-settings/settings.c
+++ b/src/lib-settings/settings.c
@@ -115,6 +115,11 @@ struct settings_mmap {
        void *mmap_base;
        size_t mmap_size;
 
+       uint32_t all_keys_hash_key_prefix;
+       uint32_t all_keys_hash_count;
+       const uint32_t *all_keys_hash;
+       const unsigned char *all_keys_base;
+
        struct event_filter **event_filters;
        struct event_filter **override_event_filters;
        bool *event_filters_are_groups;
@@ -450,6 +455,37 @@ settings_read_config_paths(struct settings_mmap *mmap,
        return 1;
 }
 
+static int
+settings_read_all_keys(struct settings_mmap *mmap,
+                      size_t *offset, const char **error_r)
+{
+       uint32_t size;
+       if (settings_block_read_uint32(mmap, offset, mmap->mmap_size,
+                                      "all keys size", &size,
+                                      error_r) < 0)
+               return -1;
+       mmap->all_keys_base = CONST_PTR_OFFSET(mmap->mmap_base, *offset);
+       size_t end_offset = *offset + size;
+
+       if (*offset % sizeof(uint32_t) != 0)
+               *offset += sizeof(uint32_t) - *offset % sizeof(uint32_t);
+       /* all keys */
+       if (settings_block_read_uint32(mmap, offset, end_offset,
+                                      "all keys hash key prefix",
+                                      &mmap->all_keys_hash_key_prefix,
+                                      error_r) < 0)
+               return -1;
+       if (settings_block_read_uint32(mmap, offset, end_offset,
+                                      "all keys hash nodes count",
+                                      &mmap->all_keys_hash_count,
+                                      error_r) < 0)
+               return -1;
+       mmap->all_keys_hash = CONST_PTR_OFFSET(mmap->mmap_base, *offset);
+
+       *offset = end_offset;
+       return 0;
+}
+
 static int
 settings_read_filters(struct settings_mmap *mmap, const char *service_name,
                      enum settings_read_flags flags, size_t *offset,
@@ -718,6 +754,8 @@ settings_mmap_parse(struct settings_mmap *mmap, const char *service_name,
        int ret;
        if ((ret = settings_read_config_paths(mmap, flags, &offset, error_r)) <= 0)
                return ret;
+       if (settings_read_all_keys(mmap, &offset, error_r) < 0)
+               return -1;
        if (settings_read_filters(mmap, service_name, flags, &offset,
                                  &protocols, error_r) < 0)
                return -1;