From: Timo Sirainen Date: Mon, 9 Oct 2017 12:37:51 +0000 (+0300) Subject: lib-index: Add mail_index_cache_optimization_settings X-Git-Tag: 2.2.34~278 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=bc26a51a3e85f5bb2774ee1288b0c83e02f5d4bf;p=thirdparty%2Fdovecot%2Fcore.git lib-index: Add mail_index_cache_optimization_settings --- diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c index 081e9cf807..fd3af4efd2 100644 --- a/src/doveadm/dsync/dsync-brain-mailbox.c +++ b/src/doveadm/dsync/dsync-brain-mailbox.c @@ -599,6 +599,7 @@ mailbox_cache_field_name_cmp(const struct mailbox_cache_field *f1, static void dsync_cache_fields_update(const struct dsync_mailbox *local_box, const struct dsync_mailbox *remote_box, + struct mailbox *box, struct mailbox_update *update) { ARRAY_TYPE(mailbox_cache_field) local_sorted, remote_sorted, changes; @@ -630,7 +631,8 @@ dsync_cache_fields_update(const struct dsync_mailbox *local_box, local_fields = array_get(&local_sorted, &local_count); remote_fields = array_get(&remote_sorted, &remote_count); t_array_init(&changes, local_count + remote_count); - drop_older_timestamp = ioloop_time - MAIL_CACHE_FIELD_DROP_SECS; + drop_older_timestamp = ioloop_time - + box->index->optimization_set.cache.unaccessed_field_drop_secs; for (li = ri = 0; li < local_count || ri < remote_count; ) { ret = li == local_count ? 1 : @@ -702,7 +704,7 @@ bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain, } } - dsync_cache_fields_update(local_box, remote_box, &update); + dsync_cache_fields_update(local_box, remote_box, box, &update); if (update.uid_validity == 0 && update.cache_updates == NULL) { diff --git a/src/lib-index/mail-cache-compress.c b/src/lib-index/mail-cache-compress.c index a8f5328ffa..b784765d02 100644 --- a/src/lib-index/mail-cache-compress.c +++ b/src/lib-index/mail-cache-compress.c @@ -207,7 +207,8 @@ mail_cache_copy(struct mail_cache *cache, struct mail_index_transaction *trans, used fields */ idx_hdr = mail_index_get_header(view); max_drop_time = idx_hdr->day_stamp == 0 ? 0 : - idx_hdr->day_stamp - MAIL_CACHE_FIELD_DROP_SECS; + idx_hdr->day_stamp - + cache->index->optimization_set.cache.unaccessed_field_drop_secs; orig_fields_count = cache->fields_count; if (cache->file_fields_count == 0) { @@ -272,7 +273,7 @@ mail_cache_copy(struct mail_cache *cache, struct mail_index_transaction *trans, mail_cache_compress_field(&ctx, &field); if (ctx.buffer->used == sizeof(cache_rec) || - ctx.buffer->used > MAIL_CACHE_RECORD_MAX_SIZE) { + ctx.buffer->used > cache->index->optimization_set.cache.record_max_size) { /* nothing cached */ ext_offset = 0; } else { diff --git a/src/lib-index/mail-cache-fields.c b/src/lib-index/mail-cache-fields.c index 05c3771f62..ec2ea77c21 100644 --- a/src/lib-index/mail-cache-fields.c +++ b/src/lib-index/mail-cache-fields.c @@ -287,7 +287,7 @@ mail_cache_header_fields_get_offset(struct mail_cache *cache, } cache->last_field_header_offset = offset; - if (next_count > MAIL_CACHE_HEADER_FIELD_CONTINUE_COUNT) + if (next_count > cache->index->optimization_set.cache.compress_header_continue_count) cache->need_compress_file_seq = cache->hdr->file_seq; if (field_hdr_r != NULL) { @@ -372,7 +372,8 @@ int mail_cache_header_fields_read(struct mail_cache *cache) cache->field_file_map[i] = (uint32_t)-1; max_drop_time = cache->index->map->hdr.day_stamp == 0 ? 0 : - cache->index->map->hdr.day_stamp - MAIL_CACHE_FIELD_DROP_SECS; + cache->index->map->hdr.day_stamp - + cache->index->optimization_set.cache.unaccessed_field_drop_secs; i_zero(&field); for (i = 0; i < field_hdr->fields_count; i++) { diff --git a/src/lib-index/mail-cache-private.h b/src/lib-index/mail-cache-private.h index c911fdbf40..8529f72db5 100644 --- a/src/lib-index/mail-cache-private.h +++ b/src/lib-index/mail-cache-private.h @@ -8,26 +8,6 @@ #define MAIL_CACHE_MAJOR_VERSION 1 #define MAIL_CACHE_MINOR_VERSION 1 -/* Drop fields that haven't been accessed for n seconds */ -#define MAIL_CACHE_FIELD_DROP_SECS (3600*24*30) - -/* Never compress the file if it's smaller than this */ -#define MAIL_CACHE_COMPRESS_MIN_SIZE (1024*32) - -/* Compress the file when n% of records are deleted */ -#define MAIL_CACHE_COMPRESS_DELETE_PERCENTAGE 20 - -/* Compress the file when n% of rows contain continued rows. - 200% means that there's 2 continued rows per record. */ -#define MAIL_CACHE_COMPRESS_CONTINUED_PERCENTAGE 200 - -/* Compress the file when we need to follow more than n next_offsets to find - the latest cache header. */ -#define MAIL_CACHE_HEADER_FIELD_CONTINUE_COUNT 4 - -/* If cache record becomes larger than this, don't add it. */ -#define MAIL_CACHE_RECORD_MAX_SIZE (64*1024) - #define MAIL_CACHE_LOCK_TIMEOUT 10 #define MAIL_CACHE_LOCK_CHANGE_TIMEOUT 300 diff --git a/src/lib-index/mail-cache-transaction.c b/src/lib-index/mail-cache-transaction.c index 96fedd457f..0098ab30d4 100644 --- a/src/lib-index/mail-cache-transaction.c +++ b/src/lib-index/mail-cache-transaction.c @@ -521,7 +521,7 @@ mail_cache_transaction_update_last_rec(struct mail_cache_transaction_ctx *ctx) size_t size; size = mail_cache_transaction_update_last_rec_size(ctx); - if (size > MAIL_CACHE_RECORD_MAX_SIZE) { + if (size > ctx->cache->index->optimization_set.cache.record_max_size) { buffer_set_used_size(ctx->cache_data, ctx->last_rec_pos); return; } diff --git a/src/lib-index/mail-cache.c b/src/lib-index/mail-cache.c index 701a53ba27..6658e15312 100644 --- a/src/lib-index/mail-cache.c +++ b/src/lib-index/mail-cache.c @@ -245,6 +245,8 @@ int mail_cache_reopen(struct mail_cache *cache) static void mail_cache_update_need_compress(struct mail_cache *cache) { + const struct mail_index_cache_optimization_settings *set = + &cache->index->optimization_set.cache; const struct mail_cache_header *hdr = cache->hdr; struct stat st; unsigned int msg_count; @@ -274,14 +276,14 @@ static void mail_cache_update_need_compress(struct mail_cache *cache) } cont_percentage = hdr->continued_record_count * 100 / records_count; - if (cont_percentage >= MAIL_CACHE_COMPRESS_CONTINUED_PERCENTAGE) { + if (cont_percentage >= set->compress_continued_percentage) { /* too many continued rows, compress */ want_compress = TRUE; } delete_percentage = hdr->deleted_record_count * 100 / (records_count + hdr->deleted_record_count); - if (delete_percentage >= MAIL_CACHE_COMPRESS_DELETE_PERCENTAGE) { + if (delete_percentage >= set->compress_delete_percentage) { /* too many deleted records, compress */ want_compress = TRUE; } @@ -292,7 +294,7 @@ static void mail_cache_update_need_compress(struct mail_cache *cache) mail_cache_set_syscall_error(cache, "fstat()"); return; } - if (st.st_size >= MAIL_CACHE_COMPRESS_MIN_SIZE) + if ((uoff_t)st.st_size >= set->compress_min_size) cache->need_compress_file_seq = hdr->file_seq; } diff --git a/src/lib-index/mail-index.c b/src/lib-index/mail-index.c index 164a3e4eef..84db2437a7 100644 --- a/src/lib-index/mail-index.c +++ b/src/lib-index/mail-index.c @@ -40,6 +40,14 @@ static const struct mail_index_optimization_settings default_optimization_set = .min_age_secs = 5 * 60, .log2_max_age_secs = 3600 * 24 * 2, }, + .cache = { + .unaccessed_field_drop_secs = 3600 * 24 * 30, + .record_max_size = 64 * 1024, + .compress_min_size = 32 * 1024, + .compress_delete_percentage = 20, + .compress_continued_percentage = 200, + .compress_header_continue_count = 4, + }, }; struct mail_index *mail_index_alloc(const char *dir, const char *prefix) @@ -182,6 +190,24 @@ void mail_index_set_optimization_settings(struct mail_index *index, dest->log.min_age_secs = set->log.min_age_secs; if (set->log.log2_max_age_secs != 0) dest->log.log2_max_age_secs = set->log.log2_max_age_secs; + + /* cache */ + if (set->cache.unaccessed_field_drop_secs != 0) + dest->cache.unaccessed_field_drop_secs = + set->cache.unaccessed_field_drop_secs; + if (set->cache.compress_min_size != 0) + dest->cache.compress_min_size = set->cache.compress_min_size; + if (set->cache.compress_delete_percentage != 0) + dest->cache.compress_delete_percentage = + set->cache.compress_delete_percentage; + if (set->cache.compress_continued_percentage != 0) + dest->cache.compress_continued_percentage = + set->cache.compress_continued_percentage; + if (set->cache.compress_header_continue_count != 0) + dest->cache.compress_header_continue_count = + set->cache.compress_header_continue_count; + if (set->cache.record_max_size != 0) + dest->cache.record_max_size = set->cache.record_max_size; } void mail_index_set_ext_init_data(struct mail_index *index, uint32_t ext_id, diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h index 8cc7d09066..3cc169d865 100644 --- a/src/lib-index/mail-index.h +++ b/src/lib-index/mail-index.h @@ -256,9 +256,29 @@ struct mail_index_log_optimization_settings { unsigned int log2_max_age_secs; }; +struct mail_index_cache_optimization_settings { + /* Drop fields that haven't been accessed for n seconds */ + unsigned int unaccessed_field_drop_secs; + /* If cache record becomes larger than this, don't add it. */ + unsigned int record_max_size; + + /* Never compress the file if it's smaller than this */ + uoff_t compress_min_size; + /* Compress the file when n% of records are deleted */ + unsigned int compress_delete_percentage; + /* Compress the file when n% of rows contain continued rows. + For example 200% means that the record has 2 continued rows, i.e. + it exists in 3 separate segments in the cache file. */ + unsigned int compress_continued_percentage; + /* Compress the file when we need to follow more than n next_offsets to + find the latest cache header. */ + unsigned int compress_header_continue_count; +}; + struct mail_index_optimization_settings { struct mail_index_base_optimization_settings index; struct mail_index_log_optimization_settings log; + struct mail_index_cache_optimization_settings cache; }; struct mail_index;