From 701e00a2a0cc49c20d7c05afc174404bad54e52d Mon Sep 17 00:00:00 2001 From: Timo Sirainen Date: Mon, 12 Dec 2022 01:16:21 +0200 Subject: [PATCH] lib-master, config: Change parsed config file syntax to binary Also use mmap() to parse the config more efficiently. The late doveconf exec fallback is also removed here, since mmap() isn't expected to fail and afterwards the errors are about the config file syntax errors. (The doveconf fallback is still there in the earlier code before config fd is received.) --- src/config/config-dump-full.c | 115 ++++++++++--- src/lib-master/master-service-settings.c | 197 +++++++++++++++-------- 2 files changed, 222 insertions(+), 90 deletions(-) diff --git a/src/config/config-dump-full.c b/src/config/config-dump-full.c index aef944b8c6..a627cb5e46 100644 --- a/src/config/config-dump-full.c +++ b/src/config/config-dump-full.c @@ -17,6 +17,19 @@ struct dump_context { string_t *delayed_output; }; +static void +config_dump_full_stdout_callback(const char *key, const char *value, + enum config_key_type type ATTR_UNUSED, + void *context) +{ + struct dump_context *ctx = context; + + T_BEGIN { + o_stream_nsend_str(ctx->output, t_strdup_printf( + "%s=%s\n", key, str_tabescape(value))); + } T_END; +} + static void config_dump_full_callback(const char *key, const char *value, enum config_key_type type ATTR_UNUSED, void *context) @@ -31,19 +44,22 @@ static void config_dump_full_callback(const char *key, const char *value, (suffix[0] == '\0' || suffix[0] == '/')))) { /* For backwards compatibility: global passdbs and userdbs are added after per-protocol ones, not before. */ - str_printfa(ctx->delayed_output, "%s=%s\n", key, value); - } else T_BEGIN { - o_stream_nsend_str(ctx->output, t_strdup_printf( - "%s=%s\n", key, str_tabescape(value))); - } T_END; + str_append_data(ctx->delayed_output, key, strlen(key)+1); + str_append_data(ctx->delayed_output, value, strlen(value)+1); + } else { + o_stream_nsend(ctx->output, key, strlen(key)+1); + o_stream_nsend(ctx->output, value, strlen(value)+1); + } } static void config_dump_full_write_filter(struct ostream *output, - const struct config_filter *filter) + const struct config_filter *filter, + enum config_dump_full_dest dest) { string_t *str = t_str_new(128); - str_append(str, ":FILTER "); + if (dest == CONFIG_DUMP_FULL_DEST_STDOUT) + str_append(str, ":FILTER "); unsigned int prefix_len = str_len(str); if (filter->service != NULL) { @@ -67,12 +83,17 @@ config_dump_full_write_filter(struct ostream *output, i_assert(str_len(str) > prefix_len); str_delete(str, str_len(str) - 4, 4); - str_append_c(str, '\n'); + if (dest == CONFIG_DUMP_FULL_DEST_STDOUT) + str_append_c(str, '\n'); + else + str_append_c(str, '\0'); o_stream_nsend(output, str_data(str), str_len(str)); } static bool -config_dump_full_sections(struct ostream *output, unsigned int section_idx) +config_dump_full_sections(struct ostream *output, + enum config_dump_full_dest dest, + unsigned int section_idx) { struct config_filter_parser *const *filters; struct config_export_context *export_ctx; @@ -91,13 +112,37 @@ config_dump_full_sections(struct ostream *output, unsigned int section_idx) }; for (; *filters != NULL && ret == 0; filters++) T_BEGIN { - config_dump_full_write_filter(output, &(*filters)->filter); - export_ctx = config_export_init( - CONFIG_DUMP_SCOPE_SET, - CONFIG_DUMP_FLAG_HIDE_LIST_DEFAULTS, - config_dump_full_callback, &dump_ctx); + uint64_t blob_size = 0; + uoff_t start_offset = output->offset; + if (dest != CONFIG_DUMP_FULL_DEST_STDOUT) + o_stream_nsend(output, &blob_size, sizeof(blob_size)); + + config_dump_full_write_filter(output, &(*filters)->filter, dest); + if (dest == CONFIG_DUMP_FULL_DEST_STDOUT) { + export_ctx = config_export_init( + CONFIG_DUMP_SCOPE_SET, + CONFIG_DUMP_FLAG_HIDE_LIST_DEFAULTS, + config_dump_full_stdout_callback, &dump_ctx); + } else { + export_ctx = config_export_init( + CONFIG_DUMP_SCOPE_SET, + CONFIG_DUMP_FLAG_HIDE_LIST_DEFAULTS, + config_dump_full_callback, &dump_ctx); + } config_export_parsers(export_ctx, (*filters)->parsers); ret = config_export_finish(&export_ctx, §ion_idx); + if (ret == 0 && dest != CONFIG_DUMP_FULL_DEST_STDOUT) { + /* write the filter blob size */ + blob_size = cpu64_to_be(output->offset - start_offset); + if (o_stream_pwrite(output, &blob_size, + sizeof(blob_size), + start_offset) < 0) { + i_error("o_stream_pwrite(%s) failed: %s", + o_stream_get_name(output), + o_stream_get_error(output)); + ret = -1; + } + } } T_END; return ret == 0; } @@ -107,7 +152,6 @@ int config_dump_full(enum config_dump_full_dest dest, { struct config_export_context *export_ctx; struct config_filter empty_filter; - enum config_dump_flags flags; unsigned int section_idx = 0; int fd = -1; bool failed = FALSE; @@ -116,9 +160,15 @@ int config_dump_full(enum config_dump_full_dest dest, .delayed_output = str_new(default_pool, 256), }; - flags = CONFIG_DUMP_FLAG_CHECK_SETTINGS; - export_ctx = config_export_init(CONFIG_DUMP_SCOPE_CHANGED, flags, - config_dump_full_callback, &dump_ctx); + if (dest == CONFIG_DUMP_FULL_DEST_STDOUT) { + export_ctx = config_export_init( + CONFIG_DUMP_SCOPE_CHANGED, 0, + config_dump_full_stdout_callback, &dump_ctx); + } else { + export_ctx = config_export_init( + CONFIG_DUMP_SCOPE_CHANGED, 0, + config_dump_full_callback, &dump_ctx); + } i_zero(&empty_filter); config_export_by_filter(export_ctx, &empty_filter); @@ -163,20 +213,41 @@ int config_dump_full(enum config_dump_full_dest dest, *import_environment_r = t_strdup(config_export_get_import_environment(export_ctx)); + + uint64_t blob_size = 0; + uoff_t blob_size_offset = 0; + if (dest != CONFIG_DUMP_FULL_DEST_STDOUT) { + o_stream_nsend_str(output, "DOVECOT-CONFIG\t1.0\n"); + blob_size_offset = output->offset; + o_stream_nsend(output, &blob_size, sizeof(blob_size)); + } + if (config_export_finish(&export_ctx, §ion_idx) < 0) failed = TRUE; - else - failed = !config_dump_full_sections(output, section_idx); + else if (dest != CONFIG_DUMP_FULL_DEST_STDOUT) { + blob_size = cpu64_to_be(output->offset - blob_size_offset); + if (o_stream_pwrite(output, &blob_size, sizeof(blob_size), + blob_size_offset) < 0) { + i_error("o_stream_pwrite(%s) failed: %s", + o_stream_get_name(output), + o_stream_get_error(output)); + failed = TRUE; + } + } + if (!failed) + failed = !config_dump_full_sections(output, dest, section_idx); if (dump_ctx.delayed_output != NULL && str_len(dump_ctx.delayed_output) > 0) { - o_stream_nsend_str(output, ":FILTER \n"); + uint64_t blob_size = + cpu64_to_be(sizeof(blob_size) + 1 + str_len(dump_ctx.delayed_output)); + o_stream_nsend(output, &blob_size, sizeof(blob_size)); + o_stream_nsend(output, "", 1); o_stream_nsend(output, str_data(dump_ctx.delayed_output), str_len(dump_ctx.delayed_output)); } str_free(&dump_ctx.delayed_output); - o_stream_nsend_str(output, "\n"); if (o_stream_finish(output) < 0) { i_error("write(%s) failed: %s", o_stream_get_name(output), o_stream_get_error(output)); diff --git a/src/lib-master/master-service-settings.c b/src/lib-master/master-service-settings.c index e5604d9a8c..4694073a70 100644 --- a/src/lib-master/master-service-settings.c +++ b/src/lib-master/master-service-settings.c @@ -4,11 +4,9 @@ #include "array.h" #include "event-filter.h" #include "path-util.h" -#include "istream.h" +#include "mmap-util.h" #include "fdpass.h" #include "write-full.h" -#include "str.h" -#include "strescape.h" #include "syslog-util.h" #include "eacces-error.h" #include "env-util.h" @@ -434,80 +432,139 @@ filter_string_parse_protocol(const char *filter_string, } static int -master_service_settings_read_stream(struct setting_parser_context *parser, - struct event *event, struct istream *input, - struct master_service_settings_output *output_r, - const char **error_r) +master_service_settings_read_mmap(struct setting_parser_context *parser, + struct event *event, + const unsigned char *mmap_base, + size_t mmap_size, + struct master_service_settings_output *output_r, + const char **error_r) { - const char *line, *filter_string, *error; - unsigned int linenum = 0; + /* + DOVECOT-CONFIG 1.0 + + <64bit big-endian global settings blob size> + [ key value , ... ] + + <64bit big-endian filter settings blob size> + filter_string + [ key value , ... ] + + ... more filters ... + + Settings are read until the blob size is reached. There is no + padding/alignment. The mmaped data comes from a trusted source + (if we can't trust the config, what can we trust?), so for + performance and simplicity we trust the mmaped data to be properly + NUL-terminated. If it's not, it can cause a segfault. */ ARRAY_TYPE(const_string) protocols; - bool filter_match = TRUE; - int ret; t_array_init(&protocols, 8); const struct failure_context failure_ctx = { .type = LOG_TYPE_DEBUG, }; - while ((line = i_stream_read_next_line(input)) != NULL) { - if (*line == '\0') { - /* empty line finishes it */ - if (array_count(&protocols) > 0) { - array_append_zero(&protocols); - output_r->specific_services = - array_front(&protocols); - } - return 0; + + const char *magic_prefix = "DOVECOT-CONFIG\t"; + const unsigned int magic_prefix_len = strlen(magic_prefix); + const unsigned char *eol = memchr(mmap_base, '\n', mmap_size); + if (mmap_size < magic_prefix_len || + memcmp(magic_prefix, mmap_base, magic_prefix_len) != 0 || + eol == NULL) { + *error_r = "File header doesn't begin with DOVECOT-CONFIG line"; + return -1; + } + if (mmap_base[magic_prefix_len] != '1' || + mmap_base[magic_prefix_len+1] != '.') { + *error_r = t_strdup_printf( + "Unsupported config file version '%s'", + t_strdup_until(mmap_base + magic_prefix_len, eol)); + return -1; + } + + size_t start_offset = eol - mmap_base + 1; + uoff_t offset = start_offset; + do { + /* */ + uint64_t blob_size; + if (offset + sizeof(blob_size) > mmap_size) { + *error_r = t_strdup_printf( + "Config file size too small " + "(offset=%zu, file_size=%zu)", offset, mmap_size); + return -1; + } + blob_size = be64_to_cpu_unaligned(mmap_base + offset); + if (offset + blob_size > mmap_size) { + *error_r = t_strdup_printf( + "Settings blob points outside file " + "(offset=%zu, blob_size=%"PRIu64", file_size=%zu)", + offset, blob_size, mmap_size); + return -1; } - linenum++; - if (str_begins(line, ":FILTER ", &filter_string)) { + size_t end_offset = offset + blob_size; + offset += sizeof(blob_size); + + /* */ + if (offset > start_offset + sizeof(blob_size)) { + const char *filter_string = + (const char *)mmap_base + offset; + offset += strlen(filter_string) + 1; + if (offset > end_offset) { + *error_r = t_strdup_printf( + "Filter points outside blob " + "(offset=%zu, end_offset=%zu, file_size=%zu)", + offset, end_offset, mmap_size); + return -1; + } + struct event_filter *filter = event_filter_create(); + const char *error; filter_string_parse_protocol(filter_string, &protocols); if (event_filter_parse(filter_string, filter, &error) < 0) { *error_r = t_strdup_printf( - "Line %u: Received invalid FILTER %s: %s", - linenum, filter_string, error); - ret = -1; - } else { - ret = 0; - filter_match = filter_string[0] == '\0' || - event_filter_match(filter, event, - &failure_ctx); + "Received invalid filter '%s': %s", + filter_string, error); + event_filter_unref(&filter); + return -1; } + bool match = filter_string[0] == '\0' || + event_filter_match(filter, event, + &failure_ctx); event_filter_unref(&filter); - if (ret < 0) - return -1; - continue; + if (!match) { + /* Filter didn't match. Jump to the next one. */ + offset = end_offset; + continue; + } } - if (!filter_match) - continue; - T_BEGIN { - line = t_str_tabunescape(line); - ret = settings_parse_line(parser, line); - } T_END; - - if (ret < 0) { - *error_r = t_strdup_printf("Line %u: %s", linenum, - settings_parser_get_error(parser)); - return -1; + /* list of settings: key, value, ... */ + while (offset < end_offset) { + const char *key = (const char *)mmap_base + offset; + offset += strlen(key)+1; + const char *value = (const char *)mmap_base + offset; + offset += strlen(value)+1; + if (offset > end_offset) { + *error_r = t_strdup_printf( + "Settings key/value points outside blob " + "(offset=%zu, end_offset=%zu, file_size=%zu)", + offset, end_offset, mmap_size); + return -1; + } + int ret; + T_BEGIN { + ret = settings_parse_keyvalue(parser, key, value); + if (ret < 0) + *error_r = t_strdup(settings_parser_get_error(parser)); + } T_END_PASS_STR_IF(ret < 0, error_r); + if (ret < 0) + return -1; } - } + } while (offset < mmap_size); - if (input->stream_errno != 0) { - *error_r = t_strdup_printf("read(%s) failed: %s", - i_stream_get_name(input), - i_stream_get_error(input)); - } else if (input->v_offset == 0) { - *error_r = t_strdup_printf( - "read(%s) returned EOF before receiving any data", - i_stream_get_name(input)); - } else { - *error_r = t_strdup_printf( - "read(%s) returned EOF before receiving end-of-settings line", - i_stream_get_name(input)); + if (array_count(&protocols) > 0) { + array_append_zero(&protocols); + output_r->specific_services = array_front(&protocols); } - return -1; + return 0; } int master_service_settings_read(struct master_service *service, @@ -518,7 +575,6 @@ int master_service_settings_read(struct master_service *service, ARRAY(const struct setting_parser_info *) all_roots; const struct setting_parser_info *tmp_root; struct setting_parser_context *parser; - struct istream *istream; const char *path = NULL, *value, *error; unsigned int i; int ret, fd = -1; @@ -529,8 +585,6 @@ int master_service_settings_read(struct master_service *service, if (service->config_fd != -1 && !input->reload_config) { /* config was already read once */ fd = service->config_fd; - if (lseek(fd, 0, SEEK_SET) < 0) - i_fatal("lseek(config fd) failed: %m"); path = ""; } else if ((value = getenv(DOVECOT_CONFIG_FD_ENV)) != NULL) { /* doveconf -F parameter already executed us back. @@ -584,20 +638,27 @@ int master_service_settings_read(struct master_service *service, array_front(&all_roots), array_count(&all_roots), SETTINGS_PARSER_FLAG_IGNORE_UNKNOWN_KEYS); + /* fd is unset only if MASTER_SERVICE_FLAG_NO_CONFIG_SETTINGS is used */ if (fd != -1) { - istream = i_stream_create_fd(fd, SIZE_MAX); - i_stream_set_name(istream, path); - ret = master_service_settings_read_stream(parser, event, - istream, output_r, error_r); - i_stream_unref(&istream); + size_t mmap_size; + void *mmap_base = mmap_ro_file(fd, &mmap_size); + if (mmap_base == MAP_FAILED) + i_fatal("mmap(%s) failed: %m", path); + if (mmap_size == 0) + i_fatal("%s file size is empty", path); + + ret = master_service_settings_read_mmap(parser, event, + mmap_base, mmap_size, + output_r, error_r); + if (munmap(mmap_base, mmap_size) < 0) + i_fatal("munmap(%s) failed: %m", path); if (ret < 0) { if (getenv(DOVECOT_CONFIG_FD_ENV) != NULL) { - i_fatal("Failed to read config from fd %d: %s", + i_fatal("Failed to parse config from fd %d: %s", fd, *error_r); } i_close_fd(&fd); - config_exec_fallback(service, input, error_r); settings_parser_unref(&parser); event_unref(&event); return -1; -- 2.47.3