From: Timo Sirainen Date: Mon, 7 Apr 2025 10:59:12 +0000 (+0300) Subject: config, lib-settings: Write all setting keys into hash table in binary config X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a1603cd875719f774c6bc0a48b90af8b50929452;p=thirdparty%2Fdovecot%2Fcore.git config, lib-settings: Write all setting keys into hash table in binary config --- diff --git a/src/config/config-dump-full.c b/src/config/config-dump-full.c index 80af484f8c..b87eeab8a6 100644 --- a/src/config/config-dump-full.c +++ b/src/config/config-dump-full.c @@ -2,8 +2,10 @@ #include "lib.h" #include "array.h" +#include "hash.h" #include "str.h" #include "strescape.h" +#include "primes.h" #include "wildcard-match.h" #include "safe-mkstemp.h" #include "ostream.h" @@ -12,6 +14,7 @@ #include "config-parser.h" #include "config-request.h" #include "config-filter.h" +#include "all-settings.h" #include "config-dump-full.h" #include @@ -37,6 +40,9 @@ <32bit: ctime UNIX timestamp> <32bit: ctime nsecs> + <32bit: all setting keys size> + + <32bit: event filter strings count> Repeat for "event filter strings count": @@ -84,6 +90,30 @@ file. This automatically causes the more specific filters to be written after the less specific ones. + All Setting Keys + ---------------- + + This hash table contains all the setting names and their details. It is + needed for handling setting overrides in an efficient way. It is guaranteed + that there are no collisions in the written hash table. + + The contents are: + + [base offset for nodes] + <0..3 bytes to pad to 32bit offset> + <32bit: hash table key prefix> + <32bit: hash table nodes count> + <32bit: setting node relative offset> + <32bit: setting block names count> + <32bit: setting block name relative offset> + + The contents are: + + + <32bit: enum setting_type> + <32bit: setting blocks count> + <32bit: setting block name index> + Groups ------ @@ -196,6 +226,157 @@ config_dump_full_write_cache_paths(struct ostream *output, main_path->path); } +static bool +config_dump_keys_hash_try(struct config_parsed *config, + uint32_t key_prefix, unsigned int hash_table_size) +{ + const HASH_TABLE_TYPE(config_key) all_keys = + config_parsed_get_all_keys(config); + const char *name; + struct config_parser_key *config_key; + bool ret = TRUE; + + bool *table = i_new(bool, hash_table_size); + struct hash_iterate_context *iter = hash_table_iterate_init(all_keys); + while (hash_table_iterate(iter, all_keys, &name, &config_key)) { + uint32_t key_hash = (key_prefix ^ str_hash(name)) % + hash_table_size; + if (table[key_hash]) { + ret = FALSE; + break; + } + table[key_hash] = TRUE; + } + hash_table_iterate_deinit(&iter); + i_free(table); + return ret; +} + +static unsigned int +config_parser_key_list_count(struct config_parser_key *config_key) +{ + unsigned int i; + for (i = 0; config_key != NULL; i++) + config_key = config_key->next; + return i; +} + +static void +config_dump_keys_hash(struct config_parsed *config, buffer_t *buf, + uint32_t key_prefix, uint32_t hash_table_size) +{ + const HASH_TABLE_TYPE(config_key) all_keys = + config_parsed_get_all_keys(config); + const char *name; + struct config_parser_key *config_key; + + /* hash table key prefix */ + buffer_append(buf, &key_prefix, sizeof(key_prefix)); + /* hash table nodes count */ + buffer_append(buf, &hash_table_size, sizeof(hash_table_size)); + /* reserve space for the hash table */ + size_t hash_table_offset = buf->used; + buffer_append_zero(buf, sizeof(uint32_t) * hash_table_size); + + /* setting block names */ + uint32_t count; + for (count = 0; all_infos[count] != NULL; count++) ; + buffer_append(buf, &count, sizeof(count)); + size_t blocks_offset = buf->used; + buffer_append_zero(buf, sizeof(uint32_t) * count); + for (uint32_t i = 0; i < count; i++) { + uint32_t offset = buf->used; + buffer_write(buf, blocks_offset + i * sizeof(uint32_t), + &offset, sizeof(offset)); + buffer_append(buf, all_infos[i]->name, + strlen(all_infos[i]->name) + 1); + } + + /* setting nodes */ + uint32_t *hash_table = i_new(uint32_t, hash_table_size); + struct hash_iterate_context *iter = hash_table_iterate_init(all_keys); + while (hash_table_iterate(iter, all_keys, &name, &config_key)) { + uint32_t key_hash = (key_prefix ^ str_hash(name)) % + hash_table_size; + i_assert(hash_table[key_hash] == 0); + + hash_table[key_hash] = buf->used; + buffer_append(buf, name, strlen(name) + 1); + + const struct setting_define *def = + &all_infos[config_key->info_idx]->defines[config_key->define_idx]; + uint32_t num32 = def->type; + buffer_append(buf, &num32, sizeof(num32)); + + num32 = config_parser_key_list_count(config_key); + buffer_append(buf, &num32, sizeof(num32)); + + do { + num32 = config_key->info_idx; + buffer_append(buf, &num32, sizeof(num32)); + config_key = config_key->next; + } while (config_key != NULL); + } + hash_table_iterate_deinit(&iter); + + buffer_write(buf, hash_table_offset, hash_table, + sizeof(uint32_t) * hash_table_size); + i_free(hash_table); +} + +static void +config_dump_keys_hash_table(struct config_parsed *config, buffer_t *buf) +{ + const HASH_TABLE_TYPE(config_key) all_keys = + config_parsed_get_all_keys(config); + unsigned int i, hash_table_size = hash_table_count(all_keys); + + /* Find a hash table where no settings keys have collisions. We can + spend more time here so later settings lookups are more efficient by + not having to handle key collisions. */ + for (;;) { + /* Try a few times to fit the hash table into a smaller node + count by using different key prefixes. Use predictable key + prefixes, so the hash table sizes are also predictable (and + easier to debug) across different runs. */ + for (i = 0; i < 10; i++) { + uint32_t key_prefix = i; + if (config_dump_keys_hash_try(config, key_prefix, + hash_table_size)) { + config_dump_keys_hash(config, buf, + key_prefix, hash_table_size); + return; + } + } + /* Continue with a larger hash table size until we succeed. + In theory this could cause huge hash table sizes (or even + out of memory), but practically that shouldn't happen. */ + hash_table_size = primes_closest(hash_table_size + 1); + } +} + +static void +config_dump_full_write_all_keys(struct ostream *output, + struct config_parsed *config) +{ + buffer_t *buf = buffer_create_dynamic(default_pool, 10240); + + /* 32bit padding */ + if (output->offset % sizeof(uint32_t) != 0) { + buffer_append_zero(buf, sizeof(uint32_t) - + output->offset % sizeof(uint32_t)); + } + config_dump_keys_hash_table(config, buf); + + /* all setting keys size */ + i_assert(buf->used <= UINT32_MAX); + uint32_t set_size = buf->used; + + o_stream_nsend(output, &set_size, sizeof(set_size)); + o_stream_nsend(output, buf->data, buf->used); + buffer_free(&buf); +} + static void config_dump_full_append_filter_query(string_t *str, const struct config_filter *filter, @@ -762,6 +943,7 @@ int config_dump_full(struct config_parsed *config, &cache_path); if (cache_path != NULL) final_path = cache_path; + config_dump_full_write_all_keys(output, config); config_dump_full_write_filters(output, config); } diff --git a/src/lib-master/test-master-service-settings.c b/src/lib-master/test-master-service-settings.c index d3a614fd14..ff2f11fee2 100644 --- a/src/lib-master/test-master-service-settings.c +++ b/src/lib-master/test-master-service-settings.c @@ -44,25 +44,66 @@ static const struct { "\x00\x00\x00"), // cache path count "Area too small when reading uint of 'config paths count'" }, - /* event filter count is truncated */ + /* all keys size is truncated */ { DATA("DOVECOT-CONFIG\t1.0\n" NUM64("\x07") // full size NUM32("\x00") // cache path count + "\x00\x00\x00"), // all keys size + "Area too small when reading uint of 'all keys size'" }, + + /* all keys hash key prefix is truncated */ + { DATA("DOVECOT-CONFIG\t1.0\n" + NUM64("\x0C") // full size + NUM32("\x00") // cache path count + NUM32("\x04") // all keys size + "\x00" // 32bit padding + "\x00\x00\x00"), // all keys hash key prefix + "Area too small when reading uint of 'all keys hash key prefix'" }, + + /* event all keys hash nodes count is truncated */ + { DATA("DOVECOT-CONFIG\t1.0\n" + NUM64("\x10") // full size + NUM32("\x00") // cache path count + NUM32("\x08") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + "\x00\x00\x00"), // all keys hash nodes count + "Area too small when reading uint of 'all keys hash nodes count'" }, + + /* event filter count is truncated */ + { DATA("DOVECOT-CONFIG\t1.0\n" + NUM64("\x18") // full size + NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count "\x00\x00\x00"), // event filter count "Area too small when reading uint of 'filters count'" }, /* event filter strings are truncated */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x08") // full size + NUM64("\x19") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01")), // event filter count "'filter string' points outside area" }, /* full file size is 7 bytes, which makes the first block size truncated, since it needs 8 bytes */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x11") // full size + NUM64("\x22") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -70,8 +111,13 @@ static const struct { "Area too small when reading size of 'block size'" }, /* first block size is 0, which is too small */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x12") // full size + NUM64("\x23") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -79,8 +125,13 @@ static const struct { "'block name' points outside area" }, /* first block size is 1, but full file size is too small */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x12") // full size + NUM64("\x23") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -88,8 +139,13 @@ static const struct { "'block size' points outside are" }, /* block name is not NUL-terminated */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x14") // full size + NUM64("\x25") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -100,8 +156,13 @@ static const struct { /* settings count is truncated */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x17") // full size + NUM64("\x28") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -112,8 +173,13 @@ static const struct { /* settings keys are truncated */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x18") // full size + NUM64("\x29") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -124,8 +190,13 @@ static const struct { /* filter count is truncated */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x1D") // full size + NUM64("\x2E") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -138,8 +209,13 @@ static const struct { /* filter settings size is truncated */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x25") // full size + NUM64("\x36") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -153,8 +229,13 @@ static const struct { /* filter settings is truncated */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x26") // full size + NUM64("\x37") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -167,8 +248,13 @@ static const struct { "'filter settings size' points outside area" }, /* filter error is missing */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x33") // full size + NUM64("\x44") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -184,8 +270,13 @@ static const struct { "'filter error string' points outside area" }, /* filter error is not NUL-terminated */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x41") // full size + NUM64("\x52") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -202,8 +293,13 @@ static const struct { "'filter error string' points outside area" }, /* include group count is truncated */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x44") // full size + NUM64("\x55") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -221,8 +317,13 @@ static const struct { "Area too small when reading uint of 'include group count'" }, /* include group count is too large */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x45") // full size + NUM64("\x56") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -240,8 +341,13 @@ static const struct { "'group label string' points outside area" }, /* group label not NUL-terminated */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x46") // full size + NUM64("\x57") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -260,8 +366,13 @@ static const struct { "'group label string' points outside area" }, /* group name not NUL-terminated */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x48") // full size + NUM64("\x59") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] @@ -281,8 +392,13 @@ static const struct { "'group name string' points outside area" }, /* invalid filter string */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x3A") // full size + NUM64("\x4B") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "F\x00" // event filter[0] "F\x00" // override event filter[0] @@ -302,12 +418,17 @@ static const struct { /* Duplicate block name */ { DATA("DOVECOT-CONFIG\t1.0\n" - NUM64("\x44") // full size + NUM64("\x54") // full size NUM32("\x00") // cache path count + NUM32("\x0D") // all keys size + "\x00" // 32bit padding + NUM32("\x00") // all keys hash key prefix + NUM32("\x00") // all keys hash nodes count + NUM32("\x00") // block names count NUM32("\x01") // event filter count "\x00" // event filter[0] "\x00" // override event filter[0] - NUM64("\x28") // block size + NUM64("\x27") // block size "N\x00" // block name NUM32("\x01") // settings count "K\x00" // setting[0] key @@ -315,7 +436,7 @@ static const struct { NUM64("\x05") // filter settings size "\x00" // filter error string NUM32("\x00") // include group count - "\x00\x00" // 64bit padding + "\x00" // 64bit padding NUM64("\x00") // filter[0] settings offset NUM32("\x00") // filter[0] event filter index "\x00" // safety NUL diff --git a/src/lib-settings/settings.c b/src/lib-settings/settings.c index 3f75cea7d6..efeced1d54 100644 --- a/src/lib-settings/settings.c +++ b/src/lib-settings/settings.c @@ -115,6 +115,11 @@ struct settings_mmap { void *mmap_base; size_t mmap_size; + uint32_t all_keys_hash_key_prefix; + uint32_t all_keys_hash_count; + const uint32_t *all_keys_hash; + const unsigned char *all_keys_base; + struct event_filter **event_filters; struct event_filter **override_event_filters; bool *event_filters_are_groups; @@ -450,6 +455,37 @@ settings_read_config_paths(struct settings_mmap *mmap, return 1; } +static int +settings_read_all_keys(struct settings_mmap *mmap, + size_t *offset, const char **error_r) +{ + uint32_t size; + if (settings_block_read_uint32(mmap, offset, mmap->mmap_size, + "all keys size", &size, + error_r) < 0) + return -1; + mmap->all_keys_base = CONST_PTR_OFFSET(mmap->mmap_base, *offset); + size_t end_offset = *offset + size; + + if (*offset % sizeof(uint32_t) != 0) + *offset += sizeof(uint32_t) - *offset % sizeof(uint32_t); + /* all keys */ + if (settings_block_read_uint32(mmap, offset, end_offset, + "all keys hash key prefix", + &mmap->all_keys_hash_key_prefix, + error_r) < 0) + return -1; + if (settings_block_read_uint32(mmap, offset, end_offset, + "all keys hash nodes count", + &mmap->all_keys_hash_count, + error_r) < 0) + return -1; + mmap->all_keys_hash = CONST_PTR_OFFSET(mmap->mmap_base, *offset); + + *offset = end_offset; + return 0; +} + static int settings_read_filters(struct settings_mmap *mmap, const char *service_name, enum settings_read_flags flags, size_t *offset, @@ -718,6 +754,8 @@ settings_mmap_parse(struct settings_mmap *mmap, const char *service_name, int ret; if ((ret = settings_read_config_paths(mmap, flags, &offset, error_r)) <= 0) return ret; + if (settings_read_all_keys(mmap, &offset, error_r) < 0) + return -1; if (settings_read_filters(mmap, service_name, flags, &offset, &protocols, error_r) < 0) return -1;