From: Marco Bettini Date: Tue, 6 Dec 2022 11:32:13 +0000 (+0000) Subject: global: Add mail_cache_max_header_name_length functionality X-Git-Tag: 2.4.0~3268 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3a12936c37c4d46d039af7cb73ecd2588b4bf660;p=thirdparty%2Fdovecot%2Fcore.git global: Add mail_cache_max_header_name_length functionality --- diff --git a/src/lib-index/mail-cache-decisions.c b/src/lib-index/mail-cache-decisions.c index 7d8baf7bf1..a725ff640d 100644 --- a/src/lib-index/mail-cache-decisions.c +++ b/src/lib-index/mail-cache-decisions.c @@ -279,7 +279,8 @@ int mail_cache_decisions_copy(struct mail_cache *src, struct mail_cache *dst) mail_cache_register_get_list(src, pool_datastack_create(), &count); i_assert(fields != NULL || count == 0); if (count > 0) - mail_cache_register_fields(dst, fields, count); + mail_cache_register_fields(dst, fields, count, + unsafe_data_stack_pool); /* Destination cache isn't expected to exist yet, so use purging to create it. Setting field_header_write_pending also guarantees diff --git a/src/lib-index/mail-cache-fields.c b/src/lib-index/mail-cache-fields.c index 9a2f2c1af8..4f977ad1e8 100644 --- a/src/lib-index/mail-cache-fields.c +++ b/src/lib-index/mail-cache-fields.c @@ -106,13 +106,34 @@ mail_cache_field_update(struct mail_cache *cache, void mail_cache_register_fields(struct mail_cache *cache, struct mail_cache_field *fields, - unsigned int fields_count) + unsigned int fields_count, + pool_t fields_modify_pool) { char *name; void *value; unsigned int new_idx; unsigned int i, j, registered_count; + struct mail_index_cache_optimization_settings *set = + &cache->index->optimization_set.cache; + + if (set->max_header_name_length > 0) { + if (fields_modify_pool == NULL) + fields_modify_pool = cache->field_pool; + + unsigned int maxlen = strlen("hdr.") + set->max_header_name_length; + for (i = 0; i < fields_count; i++) { + if (fields[i].type == MAIL_CACHE_FIELD_HEADER && + strlen(fields[i].name) > maxlen) { + i_assert(fields_modify_pool != + MAIL_CACHE_TRUNCATE_NAME_FAIL); + fields[i].name = p_strndup( + fields_modify_pool, + fields[i].name, maxlen); + } + } + } + new_idx = cache->fields_count; for (i = 0; i < fields_count; i++) { if (hash_table_lookup_full(cache->field_name_hash, @@ -177,6 +198,16 @@ mail_cache_register_lookup(struct mail_cache *cache, const char *name) char *key; void *value; + struct mail_index_cache_optimization_settings *set = + &cache->index->optimization_set.cache; + + if (set->max_header_name_length > 0) { + unsigned int maxlen = strlen("hdr.") + set->max_header_name_length; + if (str_begins_icase_with(name, "hdr.") && + strlen(name) > maxlen) + name = t_strndup(name, maxlen); + } + if (hash_table_lookup_full(cache->field_name_hash, name, &key, &value)) return POINTER_CAST_TO(value, unsigned int); else @@ -434,7 +465,8 @@ int mail_cache_header_fields_read(struct mail_cache *cache) field.type = types[i]; field.field_size = sizes[i]; field.decision = file_dec; - mail_cache_register_fields(cache, &field, 1); + mail_cache_register_fields(cache, &field, 1, + cache->field_pool); fidx = field.idx; } if (cache->field_file_map[fidx] != (uint32_t)-1) { diff --git a/src/lib-index/mail-cache.h b/src/lib-index/mail-cache.h index 3a0c3db1b1..88ab2ad423 100644 --- a/src/lib-index/mail-cache.h +++ b/src/lib-index/mail-cache.h @@ -69,12 +69,17 @@ struct mail_cache * mail_cache_open_or_create_path(struct mail_index *index, const char *path); void mail_cache_free(struct mail_cache **cache); -/* Register fields. fields[].idx is updated to contain field index. - If field already exists and its caching decision is NO, the decision is - updated to the input field's decision. */ +/* Register fields. fields[].idx is updated to contain field index. If field + already exists and its caching decision is NO, the decision is updated to + the input field's decision. + + If a header field name is too long, a truncated version replaces the original + in the passed fields array. The new string is from the passed pool */ +#define MAIL_CACHE_TRUNCATE_NAME_FAIL NULL void mail_cache_register_fields(struct mail_cache *cache, struct mail_cache_field *fields, - unsigned int fields_count); + unsigned int fields_count, + pool_t fields_modify_pool); /* Returns registered field index, or UINT_MAX if not found. */ unsigned int mail_cache_register_lookup(struct mail_cache *cache, const char *name); diff --git a/src/lib-index/mail-index.c b/src/lib-index/mail-index.c index 01bb93348d..1d880e9e7b 100644 --- a/src/lib-index/mail-index.c +++ b/src/lib-index/mail-index.c @@ -227,6 +227,7 @@ void mail_index_set_optimization_settings(struct mail_index *index, if (set->cache.record_max_size != 0) dest->cache.record_max_size = set->cache.record_max_size; + dest->cache.max_header_name_length = set->cache.max_header_name_length; dest->cache.max_headers_count = set->cache.max_headers_count; } diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h index c401fc679e..8d5bb2dcca 100644 --- a/src/lib-index/mail-index.h +++ b/src/lib-index/mail-index.h @@ -331,6 +331,8 @@ struct mail_index_cache_optimization_settings { /* If cache record becomes larger than this, don't add it. */ unsigned int record_max_size; + /* Maximum length for cacheable headers */ + unsigned int max_header_name_length; /* Maximum number of headers to cache */ unsigned int max_headers_count; diff --git a/src/lib-index/test-mail-cache-common.c b/src/lib-index/test-mail-cache-common.c index ee34c04fcd..b6620b728d 100644 --- a/src/lib-index/test-mail-cache-common.c +++ b/src/lib-index/test-mail-cache-common.c @@ -35,14 +35,15 @@ void test_mail_cache_init(struct mail_index *index, by randomizing the registration order. This only works for the 2nd index that is opened, because the initial cache is always created with all cache fields in the same order. */ + pool_t pool = MAIL_CACHE_TRUNCATE_NAME_FAIL; if (i_rand_limit(2) == 0) { - mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field, 1); - mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field2, 1); - mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field3, 1); + mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field, 1, pool); + mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field2, 1, pool); + mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field3, 1, pool); } else { - mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field3, 1); - mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field2, 1); - mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field, 1); + mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field3, 1, pool); + mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field2, 1, pool); + mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field, 1, pool); } } diff --git a/src/lib-index/test-mail-cache-fields.c b/src/lib-index/test-mail-cache-fields.c index 18b6a339b2..1b76dc5c81 100644 --- a/src/lib-index/test-mail-cache-fields.c +++ b/src/lib-index/test-mail-cache-fields.c @@ -23,11 +23,13 @@ static void test_mail_cache_fields_read_write(void) test_begin("mail cache fields read-write"); test_mail_cache_init(test_mail_index_init(), &ctx); - mail_cache_register_fields(ctx.cache, &cache_field, 1); + mail_cache_register_fields(ctx.cache, &cache_field, 1, + unsafe_data_stack_pool); test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); /* after writing the initial cache file, register another cache field that doesn't exist in it. */ - mail_cache_register_fields(ctx.cache, &cache_field2, 1); + mail_cache_register_fields(ctx.cache, &cache_field2, 1, + unsafe_data_stack_pool); struct mail_cache_field_private *priv = &ctx.cache->fields[cache_field.idx]; diff --git a/src/lib-index/test-mail-cache-purge.c b/src/lib-index/test-mail-cache-purge.c index 54086af0dd..ee0b33803a 100644 --- a/src/lib-index/test-mail-cache-purge.c +++ b/src/lib-index/test-mail-cache-purge.c @@ -707,7 +707,8 @@ static void test_mail_cache_purge_field_changes_int(enum test_drop drop) by purging (which is called to auto-create the cache). */ test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); mail_cache_register_fields(ctx.cache, cache_fields, - N_ELEMENTS(cache_fields)); + N_ELEMENTS(cache_fields), + unsafe_data_stack_pool); trans = mail_index_transaction_begin(ctx.view, 0); cache_view = mail_cache_view_open(ctx.cache, ctx.view); @@ -891,7 +892,8 @@ static void test_mail_cache_purge_bitmask(void) test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0); - mail_cache_register_fields(ctx.cache, &bitmask_field, 1); + mail_cache_register_fields(ctx.cache, &bitmask_field, 1, + unsafe_data_stack_pool); test_mail_cache_update_day_first_uid7(&ctx, 3); diff --git a/src/lib-index/test-mail-cache.c b/src/lib-index/test-mail-cache.c index 9938e737c9..9c279097ff 100644 --- a/src/lib-index/test-mail-cache.c +++ b/src/lib-index/test-mail-cache.c @@ -108,7 +108,8 @@ static void test_mail_cache_fields(void) test_begin("mail cache uncommitted lookups"); test_mail_cache_init(test_mail_index_init(), &ctx); mail_cache_register_fields(ctx.cache, cache_fields, - N_ELEMENTS(cache_fields)); + N_ELEMENTS(cache_fields), + unsafe_data_stack_pool); test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); @@ -401,7 +402,8 @@ static void test_mail_cache_add_decisions(void) test_mail_cache_init(test_mail_index_init(), &ctx); memcpy(cache_fields, decision_cache_fields, sizeof(cache_fields)); - mail_cache_register_fields(ctx.cache, cache_fields, TEST_FIELD_COUNT); + mail_cache_register_fields(ctx.cache, cache_fields, TEST_FIELD_COUNT, + unsafe_data_stack_pool); for (i = 0; i < TEST_FIELD_COUNT; i++) expected_decisions[i] = cache_fields[i].decision; @@ -478,7 +480,8 @@ static void test_mail_cache_lookup_decisions_int(bool header_lookups) /* register fields after the initial purge create the cache */ memcpy(cache_fields, decision_cache_fields, sizeof(cache_fields)); - mail_cache_register_fields(ctx.cache, cache_fields, TEST_FIELD_COUNT); + mail_cache_register_fields(ctx.cache, cache_fields, TEST_FIELD_COUNT, + unsafe_data_stack_pool); for (i = 0; i < TEST_FIELD_COUNT; i++) { expected_decisions[i] = cache_fields[i].decision; expected_uid_highwater[i] = 0; @@ -767,7 +770,8 @@ static void test_mail_cache_duplicate_fields(void) test_begin("mail cache duplicate fields"); test_mail_cache_init(test_mail_index_init(), &ctx); mail_cache_register_fields(ctx.cache, cache_fields, - N_ELEMENTS(cache_fields)); + N_ELEMENTS(cache_fields), + unsafe_data_stack_pool); test_mail_cache_add_mail(&ctx, UINT_MAX, NULL); diff --git a/src/lib-storage/index/index-mail-headers.c b/src/lib-storage/index/index-mail-headers.c index 2348fb1582..a39392d5de 100644 --- a/src/lib-storage/index/index-mail-headers.c +++ b/src/lib-storage/index/index-mail-headers.c @@ -157,7 +157,8 @@ get_header_field_idx(struct mailbox *box, const char *field) header_field.decision = MAIL_CACHE_DECISION_NO; T_BEGIN { header_field.name = t_strconcat("hdr.", field, NULL); - mail_cache_register_fields(box->cache, &header_field, 1); + mail_cache_register_fields(box->cache, &header_field, 1, + unsafe_data_stack_pool); } T_END; return header_field.idx; } diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c index a8cdda5227..7590394b63 100644 --- a/src/lib-storage/index/index-storage.c +++ b/src/lib-storage/index/index-storage.c @@ -73,7 +73,8 @@ static void set_cache_decisions(struct mailbox *box, } field.decision = dec; - mail_cache_register_fields(cache, &field, 1); + mail_cache_register_fields(cache, &field, 1, + unsafe_data_stack_pool); } } @@ -85,7 +86,8 @@ static void index_cache_register_defaults(struct mailbox *box) ibox->cache_fields = index_mail_global_cache_fields_dup(); mail_cache_register_fields(cache, ibox->cache_fields, - MAIL_INDEX_CACHE_FIELD_COUNT); + MAIL_INDEX_CACHE_FIELD_COUNT, + MAIL_CACHE_TRUNCATE_NAME_FAIL); if (strcmp(set->mail_never_cache_fields, "*") == 0) { /* all caching disabled for now */ @@ -289,6 +291,7 @@ int index_storage_mailbox_alloc_index(struct mailbox *box) .cache = { .unaccessed_field_drop_secs = set->mail_cache_unaccessed_field_drop, .record_max_size = set->mail_cache_record_max_size, + .max_header_name_length = set->mail_cache_max_header_name_length, .max_headers_count = set->mail_cache_max_headers_count, .max_size = set->mail_cache_max_size, .purge_min_size = set->mail_cache_purge_min_size, @@ -518,7 +521,8 @@ index_storage_mailbox_update_cache(struct mailbox *box, if (array_count(&new_fields) > 0) { mail_cache_register_fields(box->cache, array_front_modifiable(&new_fields), - array_count(&new_fields)); + array_count(&new_fields), + unsafe_data_stack_pool); } } diff --git a/src/lib-storage/mail-storage-settings.c b/src/lib-storage/mail-storage-settings.c index 1e38e476dc..cbe49fa052 100644 --- a/src/lib-storage/mail-storage-settings.c +++ b/src/lib-storage/mail-storage-settings.c @@ -46,6 +46,7 @@ static const struct setting_define mail_storage_setting_defines[] = { DEF(STR, mail_server_admin), DEF(TIME_HIDDEN, mail_cache_unaccessed_field_drop), DEF(SIZE_HIDDEN, mail_cache_record_max_size), + DEF(UINT_HIDDEN, mail_cache_max_header_name_length), DEF(UINT_HIDDEN, mail_cache_max_headers_count), DEF(SIZE_HIDDEN, mail_cache_max_size), DEF(UINT_HIDDEN, mail_cache_min_mail_count), @@ -104,6 +105,7 @@ const struct mail_storage_settings mail_storage_default_settings = { .mail_cache_min_mail_count = 0, .mail_cache_unaccessed_field_drop = 60*60*24*30, .mail_cache_record_max_size = 64 * 1024, + .mail_cache_max_header_name_length = 100, .mail_cache_max_headers_count = 100, .mail_cache_max_size = 1024 * 1024 * 1024, .mail_cache_purge_min_size = 32 * 1024, diff --git a/src/lib-storage/mail-storage-settings.h b/src/lib-storage/mail-storage-settings.h index eacc4095c3..15361584d9 100644 --- a/src/lib-storage/mail-storage-settings.h +++ b/src/lib-storage/mail-storage-settings.h @@ -27,6 +27,7 @@ struct mail_storage_settings { unsigned int mail_cache_min_mail_count; unsigned int mail_cache_unaccessed_field_drop; uoff_t mail_cache_record_max_size; + unsigned int mail_cache_max_header_name_length; unsigned int mail_cache_max_headers_count; uoff_t mail_cache_max_size; uoff_t mail_cache_purge_min_size; diff --git a/src/lib-storage/mailbox-header.c b/src/lib-storage/mailbox-header.c index b276fc9bfd..13b2be7ddd 100644 --- a/src/lib-storage/mailbox-header.c +++ b/src/lib-storage/mailbox-header.c @@ -40,7 +40,8 @@ mailbox_header_lookup_init_real(struct mailbox *box, fields[j++] = header_field; } count = j; - mail_cache_register_fields(box->cache, fields, count); + mail_cache_register_fields(box->cache, fields, count, + unsafe_data_stack_pool); pool = pool_alloconly_create("mailbox_header_lookup_ctx", 1024); ctx = p_new(pool, struct mailbox_header_lookup_ctx, 1); diff --git a/src/plugins/pop3-migration/pop3-migration-plugin.c b/src/plugins/pop3-migration/pop3-migration-plugin.c index ff9b025793..f81f37ba05 100644 --- a/src/plugins/pop3-migration/pop3-migration-plugin.c +++ b/src/plugins/pop3-migration/pop3-migration-plugin.c @@ -254,7 +254,8 @@ static unsigned int get_cache_idx(struct mail *mail) mbox->cache_field.name = "pop3-migration.hdr"; mbox->cache_field.type = MAIL_CACHE_FIELD_FIXED_SIZE; mbox->cache_field.field_size = SHA1_RESULTLEN; - mail_cache_register_fields(mail->box->cache, &mbox->cache_field, 1); + mail_cache_register_fields(mail->box->cache, &mbox->cache_field, 1, + MAIL_CACHE_TRUNCATE_NAME_FAIL); mbox->cache_field_registered = TRUE; return mbox->cache_field.idx; }