global: Add mail_cache_max_header_name_length functionality

diff --git a/src/lib-index/mail-cache-decisions.c b/src/lib-index/mail-cache-decisions.c
index 7d8baf7bf177e32eb726ed611277b6af106931e5..a725ff640de1a4c6d282085ad357fe6054199c9a 100644 (file)
--- a/src/lib-index/mail-cache-decisions.c
+++ b/src/lib-index/mail-cache-decisions.c
@@ -279,7 +279,8 @@ int mail_cache_decisions_copy(struct mail_cache *src, struct mail_cache *dst)
                mail_cache_register_get_list(src, pool_datastack_create(), &count);
        i_assert(fields != NULL || count == 0);
        if (count > 0)
-               mail_cache_register_fields(dst, fields, count);
+               mail_cache_register_fields(dst, fields, count,
+                                          unsafe_data_stack_pool);
 
        /* Destination cache isn't expected to exist yet, so use purging
           to create it. Setting field_header_write_pending also guarantees

diff --git a/src/lib-index/mail-cache-fields.c b/src/lib-index/mail-cache-fields.c
index 9a2f2c1af812f7b03106c7a2abd8de4b28365c5c..4f977ad1e8360fad2040eab378f2a0dbe6344d5c 100644 (file)
--- a/src/lib-index/mail-cache-fields.c
+++ b/src/lib-index/mail-cache-fields.c
@@ -106,13 +106,34 @@ mail_cache_field_update(struct mail_cache *cache,
 
 void mail_cache_register_fields(struct mail_cache *cache,
                                struct mail_cache_field *fields,
-                               unsigned int fields_count)
+                               unsigned int fields_count,
+                               pool_t fields_modify_pool)
 {
        char *name;
        void *value;
        unsigned int new_idx;
        unsigned int i, j, registered_count;
 
+       struct mail_index_cache_optimization_settings *set =
+               &cache->index->optimization_set.cache;
+
+       if (set->max_header_name_length > 0) {
+               if (fields_modify_pool == NULL)
+                       fields_modify_pool = cache->field_pool;
+
+               unsigned int maxlen = strlen("hdr.") + set->max_header_name_length;
+               for (i = 0; i < fields_count; i++) {
+                       if (fields[i].type == MAIL_CACHE_FIELD_HEADER &&
+                           strlen(fields[i].name) > maxlen) {
+                               i_assert(fields_modify_pool !=
+                                        MAIL_CACHE_TRUNCATE_NAME_FAIL);
+                               fields[i].name = p_strndup(
+                                       fields_modify_pool,
+                                       fields[i].name, maxlen);
+                       }
+               }
+       }
+
        new_idx = cache->fields_count;
        for (i = 0; i < fields_count; i++) {
                if (hash_table_lookup_full(cache->field_name_hash,
@@ -177,6 +198,16 @@ mail_cache_register_lookup(struct mail_cache *cache, const char *name)
        char *key;
        void *value;
 
+       struct mail_index_cache_optimization_settings *set =
+               &cache->index->optimization_set.cache;
+
+       if (set->max_header_name_length > 0) {
+               unsigned int maxlen = strlen("hdr.") + set->max_header_name_length;
+               if (str_begins_icase_with(name, "hdr.") &&
+                   strlen(name) > maxlen)
+                       name = t_strndup(name, maxlen);
+       }
+
        if (hash_table_lookup_full(cache->field_name_hash, name, &key, &value))
                return POINTER_CAST_TO(value, unsigned int);
        else
@@ -434,7 +465,8 @@ int mail_cache_header_fields_read(struct mail_cache *cache)
                        field.type = types[i];
                        field.field_size = sizes[i];
                        field.decision = file_dec;
-                       mail_cache_register_fields(cache, &field, 1);
+                       mail_cache_register_fields(cache, &field, 1,
+                                                  cache->field_pool);
                        fidx = field.idx;
                }
                if (cache->field_file_map[fidx] != (uint32_t)-1) {

diff --git a/src/lib-index/mail-cache.h b/src/lib-index/mail-cache.h
index 3a0c3db1b1303e5f9f851dc75c42b012dcf50d7b..88ab2ad423a5e18e289a5d97e816fc6edb58b8ad 100644 (file)
--- a/src/lib-index/mail-cache.h
+++ b/src/lib-index/mail-cache.h
@@ -69,12 +69,17 @@ struct mail_cache *
 mail_cache_open_or_create_path(struct mail_index *index, const char *path);
 void mail_cache_free(struct mail_cache **cache);
 
-/* Register fields. fields[].idx is updated to contain field index.
-   If field already exists and its caching decision is NO, the decision is
-   updated to the input field's decision. */
+/* Register fields. fields[].idx is updated to contain field index. If field
+   already exists and its caching decision is NO, the decision is updated to
+   the input field's decision.
+
+   If a header field name is too long, a truncated version replaces the original
+   in the passed fields array. The new string is from the passed pool */
+#define MAIL_CACHE_TRUNCATE_NAME_FAIL NULL
 void mail_cache_register_fields(struct mail_cache *cache,
                                struct mail_cache_field *fields,
-                               unsigned int fields_count);
+                               unsigned int fields_count,
+                               pool_t fields_modify_pool);
 /* Returns registered field index, or UINT_MAX if not found. */
 unsigned int
 mail_cache_register_lookup(struct mail_cache *cache, const char *name);

diff --git a/src/lib-index/mail-index.c b/src/lib-index/mail-index.c
index 01bb93348dba236bedd444314b0c8ca722feab18..1d880e9e7b5d193b31c959ad702ecf5af81c5d46 100644 (file)
--- a/src/lib-index/mail-index.c
+++ b/src/lib-index/mail-index.c
@@ -227,6 +227,7 @@ void mail_index_set_optimization_settings(struct mail_index *index,
        if (set->cache.record_max_size != 0)
                dest->cache.record_max_size = set->cache.record_max_size;
 
+       dest->cache.max_header_name_length = set->cache.max_header_name_length;
        dest->cache.max_headers_count = set->cache.max_headers_count;
 }
 

diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h
index c401fc679e447e248f82acf1d20dd7185ec146fb..8d5bb2dcca7cc0a0b9c00f9d1d74ed2c46ce9f5d 100644 (file)
--- a/src/lib-index/mail-index.h
+++ b/src/lib-index/mail-index.h
@@ -331,6 +331,8 @@ struct mail_index_cache_optimization_settings {
        /* If cache record becomes larger than this, don't add it. */
        unsigned int record_max_size;
 
+       /* Maximum length for cacheable headers */
+       unsigned int max_header_name_length;
        /* Maximum number of headers to cache */
        unsigned int max_headers_count;
 

diff --git a/src/lib-index/test-mail-cache-common.c b/src/lib-index/test-mail-cache-common.c
index ee34c04fcd8d3b96679123888b1ed9bdc2964e07..b6620b728d223f89a32a7edb93af3c7f03473e76 100644 (file)
--- a/src/lib-index/test-mail-cache-common.c
+++ b/src/lib-index/test-mail-cache-common.c
@@ -35,14 +35,15 @@ void test_mail_cache_init(struct mail_index *index,
           by randomizing the registration order. This only works for the 2nd
           index that is opened, because the initial cache is always created
           with all cache fields in the same order. */
+       pool_t pool = MAIL_CACHE_TRUNCATE_NAME_FAIL;
        if (i_rand_limit(2) == 0) {
-               mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field, 1);
-               mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field2, 1);
-               mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field3, 1);
+               mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field, 1, pool);
+               mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field2, 1, pool);
+               mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field3, 1, pool);
        } else {
-               mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field3, 1);
-               mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field2, 1);
-               mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field, 1);
+               mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field3, 1, pool);
+               mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field2, 1, pool);
+               mail_cache_register_fields(ctx_r->cache, &ctx_r->cache_field, 1, pool);
        }
 }
 

diff --git a/src/lib-index/test-mail-cache-fields.c b/src/lib-index/test-mail-cache-fields.c
index 18b6a339b2e9306d7b764d188a294454de472881..1b76dc5c8132a700e1504a6310b3e1b23276e64a 100644 (file)
--- a/src/lib-index/test-mail-cache-fields.c
+++ b/src/lib-index/test-mail-cache-fields.c
@@ -23,11 +23,13 @@ static void test_mail_cache_fields_read_write(void)
        test_begin("mail cache fields read-write");
 
        test_mail_cache_init(test_mail_index_init(), &ctx);
-       mail_cache_register_fields(ctx.cache, &cache_field, 1);
+       mail_cache_register_fields(ctx.cache, &cache_field, 1,
+                                  unsafe_data_stack_pool);
        test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0);
        /* after writing the initial cache file, register another cache field
           that doesn't exist in it. */
-       mail_cache_register_fields(ctx.cache, &cache_field2, 1);
+       mail_cache_register_fields(ctx.cache, &cache_field2, 1,
+                                  unsafe_data_stack_pool);
 
        struct mail_cache_field_private *priv =
                &ctx.cache->fields[cache_field.idx];

diff --git a/src/lib-index/test-mail-cache-purge.c b/src/lib-index/test-mail-cache-purge.c
index 54086af0dd45fe15e8721c23b85f6029433376a4..ee0b33803a05c2247a42c8c3a356bdd1c0c5b880 100644 (file)
--- a/src/lib-index/test-mail-cache-purge.c
+++ b/src/lib-index/test-mail-cache-purge.c
@@ -707,7 +707,8 @@ static void test_mail_cache_purge_field_changes_int(enum test_drop drop)
           by purging (which is called to auto-create the cache). */
        test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0);
        mail_cache_register_fields(ctx.cache, cache_fields,
-                                  N_ELEMENTS(cache_fields));
+                                  N_ELEMENTS(cache_fields),
+                                  unsafe_data_stack_pool);
 
        trans = mail_index_transaction_begin(ctx.view, 0);
        cache_view = mail_cache_view_open(ctx.cache, ctx.view);
@@ -891,7 +892,8 @@ static void test_mail_cache_purge_bitmask(void)
        test_mail_cache_add_mail(&ctx, UINT_MAX, NULL);
        test_mail_cache_add_mail(&ctx, UINT_MAX, NULL);
        test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0);
-       mail_cache_register_fields(ctx.cache, &bitmask_field, 1);
+       mail_cache_register_fields(ctx.cache, &bitmask_field, 1,
+                                  unsafe_data_stack_pool);
 
        test_mail_cache_update_day_first_uid7(&ctx, 3);
 

diff --git a/src/lib-index/test-mail-cache.c b/src/lib-index/test-mail-cache.c
index 9938e737c980a6f15e83204a64315ced54b7450c..9c279097ff2a5159009c9b348a6704c9cb893401 100644 (file)
--- a/src/lib-index/test-mail-cache.c
+++ b/src/lib-index/test-mail-cache.c
@@ -108,7 +108,8 @@ static void test_mail_cache_fields(void)
        test_begin("mail cache uncommitted lookups");
        test_mail_cache_init(test_mail_index_init(), &ctx);
        mail_cache_register_fields(ctx.cache, cache_fields,
-                                  N_ELEMENTS(cache_fields));
+                                  N_ELEMENTS(cache_fields),
+                                  unsafe_data_stack_pool);
 
        test_mail_cache_add_mail(&ctx, UINT_MAX, NULL);
 
@@ -401,7 +402,8 @@ static void test_mail_cache_add_decisions(void)
 
        test_mail_cache_init(test_mail_index_init(), &ctx);
        memcpy(cache_fields, decision_cache_fields, sizeof(cache_fields));
-       mail_cache_register_fields(ctx.cache, cache_fields, TEST_FIELD_COUNT);
+       mail_cache_register_fields(ctx.cache, cache_fields, TEST_FIELD_COUNT,
+                                  unsafe_data_stack_pool);
        for (i = 0; i < TEST_FIELD_COUNT; i++)
                expected_decisions[i] = cache_fields[i].decision;
 
@@ -478,7 +480,8 @@ static void test_mail_cache_lookup_decisions_int(bool header_lookups)
 
        /* register fields after the initial purge create the cache */
        memcpy(cache_fields, decision_cache_fields, sizeof(cache_fields));
-       mail_cache_register_fields(ctx.cache, cache_fields, TEST_FIELD_COUNT);
+       mail_cache_register_fields(ctx.cache, cache_fields, TEST_FIELD_COUNT,
+                                  unsafe_data_stack_pool);
        for (i = 0; i < TEST_FIELD_COUNT; i++) {
                expected_decisions[i] = cache_fields[i].decision;
                expected_uid_highwater[i] = 0;
@@ -767,7 +770,8 @@ static void test_mail_cache_duplicate_fields(void)
        test_begin("mail cache duplicate fields");
        test_mail_cache_init(test_mail_index_init(), &ctx);
        mail_cache_register_fields(ctx.cache, cache_fields,
-                                  N_ELEMENTS(cache_fields));
+                                  N_ELEMENTS(cache_fields),
+                                  unsafe_data_stack_pool);
 
        test_mail_cache_add_mail(&ctx, UINT_MAX, NULL);
 

diff --git a/src/lib-storage/index/index-mail-headers.c b/src/lib-storage/index/index-mail-headers.c
index 2348fb15827862cadf2de6f732e976dccaf10c5a..a39392d5de93a953c68058c9106cf9cf37f6d439 100644 (file)
--- a/src/lib-storage/index/index-mail-headers.c
+++ b/src/lib-storage/index/index-mail-headers.c
@@ -157,7 +157,8 @@ get_header_field_idx(struct mailbox *box, const char *field)
        header_field.decision = MAIL_CACHE_DECISION_NO;
        T_BEGIN {
                header_field.name = t_strconcat("hdr.", field, NULL);
-               mail_cache_register_fields(box->cache, &header_field, 1);
+               mail_cache_register_fields(box->cache, &header_field, 1,
+                                          unsafe_data_stack_pool);
        } T_END;
        return header_field.idx;
 }

diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c
index a8cdda52271e26204a6e8ae7495e55e5a382398a..7590394b635075b41686224ff69b55ebba965eba 100644 (file)
--- a/src/lib-storage/index/index-storage.c
+++ b/src/lib-storage/index/index-storage.c
@@ -73,7 +73,8 @@ static void set_cache_decisions(struct mailbox *box,
                }
 
                field.decision = dec;
-               mail_cache_register_fields(cache, &field, 1);
+               mail_cache_register_fields(cache, &field, 1,
+                                          unsafe_data_stack_pool);
        }
 }
 
@@ -85,7 +86,8 @@ static void index_cache_register_defaults(struct mailbox *box)
 
        ibox->cache_fields = index_mail_global_cache_fields_dup();
        mail_cache_register_fields(cache, ibox->cache_fields,
-                                  MAIL_INDEX_CACHE_FIELD_COUNT);
+                                  MAIL_INDEX_CACHE_FIELD_COUNT,
+                                  MAIL_CACHE_TRUNCATE_NAME_FAIL);
 
        if (strcmp(set->mail_never_cache_fields, "*") == 0) {
                /* all caching disabled for now */
@@ -289,6 +291,7 @@ int index_storage_mailbox_alloc_index(struct mailbox *box)
                .cache = {
                        .unaccessed_field_drop_secs = set->mail_cache_unaccessed_field_drop,
                        .record_max_size = set->mail_cache_record_max_size,
+                       .max_header_name_length = set->mail_cache_max_header_name_length,
                        .max_headers_count = set->mail_cache_max_headers_count,
                        .max_size = set->mail_cache_max_size,
                        .purge_min_size = set->mail_cache_purge_min_size,
@@ -518,7 +521,8 @@ index_storage_mailbox_update_cache(struct mailbox *box,
        if (array_count(&new_fields) > 0) {
                mail_cache_register_fields(box->cache,
                                           array_front_modifiable(&new_fields),
-                                          array_count(&new_fields));
+                                          array_count(&new_fields),
+                                          unsafe_data_stack_pool);
        }
 }
 

diff --git a/src/lib-storage/mail-storage-settings.c b/src/lib-storage/mail-storage-settings.c
index 1e38e476dc1e1868177c34bb962cd3a49403485c..cbe49fa0529d679f73f1d3e635bd2afa79838bba 100644 (file)
--- a/src/lib-storage/mail-storage-settings.c
+++ b/src/lib-storage/mail-storage-settings.c
@@ -46,6 +46,7 @@ static const struct setting_define mail_storage_setting_defines[] = {
        DEF(STR, mail_server_admin),
        DEF(TIME_HIDDEN, mail_cache_unaccessed_field_drop),
        DEF(SIZE_HIDDEN, mail_cache_record_max_size),
+       DEF(UINT_HIDDEN, mail_cache_max_header_name_length),
        DEF(UINT_HIDDEN, mail_cache_max_headers_count),
        DEF(SIZE_HIDDEN, mail_cache_max_size),
        DEF(UINT_HIDDEN, mail_cache_min_mail_count),
@@ -104,6 +105,7 @@ const struct mail_storage_settings mail_storage_default_settings = {
        .mail_cache_min_mail_count = 0,
        .mail_cache_unaccessed_field_drop = 60*60*24*30,
        .mail_cache_record_max_size = 64 * 1024,
+       .mail_cache_max_header_name_length = 100,
        .mail_cache_max_headers_count = 100,
        .mail_cache_max_size = 1024 * 1024 * 1024,
        .mail_cache_purge_min_size = 32 * 1024,

diff --git a/src/lib-storage/mail-storage-settings.h b/src/lib-storage/mail-storage-settings.h
index eacc4095c3ee8c3d70aa30a6f49f8b4f0a29afa6..15361584d94c9cb5ae6e034346ed38d8297daba8 100644 (file)
--- a/src/lib-storage/mail-storage-settings.h
+++ b/src/lib-storage/mail-storage-settings.h
@@ -27,6 +27,7 @@ struct mail_storage_settings {
        unsigned int mail_cache_min_mail_count;
        unsigned int mail_cache_unaccessed_field_drop;
        uoff_t mail_cache_record_max_size;
+       unsigned int mail_cache_max_header_name_length;
        unsigned int mail_cache_max_headers_count;
        uoff_t mail_cache_max_size;
        uoff_t mail_cache_purge_min_size;

diff --git a/src/lib-storage/mailbox-header.c b/src/lib-storage/mailbox-header.c
index b276fc9bfde3b1c0429d6754980564f31e34c578..13b2be7ddd970bf95799eee95fe929a19cdffee5 100644 (file)
--- a/src/lib-storage/mailbox-header.c
+++ b/src/lib-storage/mailbox-header.c
@@ -40,7 +40,8 @@ mailbox_header_lookup_init_real(struct mailbox *box,
                fields[j++] = header_field;
        }
        count = j;
-       mail_cache_register_fields(box->cache, fields, count);
+       mail_cache_register_fields(box->cache, fields, count,
+                                  unsafe_data_stack_pool);
 
        pool = pool_alloconly_create("mailbox_header_lookup_ctx", 1024);
        ctx = p_new(pool, struct mailbox_header_lookup_ctx, 1);

diff --git a/src/plugins/pop3-migration/pop3-migration-plugin.c b/src/plugins/pop3-migration/pop3-migration-plugin.c
index ff9b025793e0ea99613b11ee72af46bdacd28d9b..f81f37ba057fcca0ed121cbddc753cc0dbbe72b5 100644 (file)
--- a/src/plugins/pop3-migration/pop3-migration-plugin.c
+++ b/src/plugins/pop3-migration/pop3-migration-plugin.c
@@ -254,7 +254,8 @@ static unsigned int get_cache_idx(struct mail *mail)
        mbox->cache_field.name = "pop3-migration.hdr";
        mbox->cache_field.type = MAIL_CACHE_FIELD_FIXED_SIZE;
        mbox->cache_field.field_size = SHA1_RESULTLEN;
-       mail_cache_register_fields(mail->box->cache, &mbox->cache_field, 1);
+       mail_cache_register_fields(mail->box->cache, &mbox->cache_field, 1,
+                                  MAIL_CACHE_TRUNCATE_NAME_FAIL);
        mbox->cache_field_registered = TRUE;
        return mbox->cache_field.idx;
 }