From: Timo Sirainen Date: Fri, 3 Apr 2009 17:39:46 +0000 (-0400) Subject: dbox: Added dbox_purge_min_percentage setting. X-Git-Tag: 2.0.alpha1~1038^2~1 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=a25fb736e9fb586c87b708699ee32871e50a8110;p=thirdparty%2Fdovecot%2Fcore.git dbox: Added dbox_purge_min_percentage setting. --HG-- branch : HEAD --- diff --git a/dovecot-example.conf b/dovecot-example.conf index 6d6e055b6a..f27c444af3 100644 --- a/dovecot-example.conf +++ b/dovecot-example.conf @@ -526,6 +526,10 @@ # midnight, so 1 = today, 2 = yesterday, etc. 0 = check disabled. #dbox_rotate_days = 0 +# Don't purge a dbox file until this many % of it contains expunged messages. +# 0 purges always, 100 purges never. +#dbox_purge_min_percentage = 0 + ## ## IMAP specific settings ## diff --git a/src/lib-storage/index/dbox/dbox-map.c b/src/lib-storage/index/dbox/dbox-map.c index e4dda0dce9..723e0dcd85 100644 --- a/src/lib-storage/index/dbox/dbox-map.c +++ b/src/lib-storage/index/dbox/dbox-map.c @@ -2,6 +2,7 @@ #include "lib.h" #include "array.h" +#include "hash.h" #include "ostream.h" #include "mkdir-parents.h" #include "dbox-storage.h" @@ -241,6 +242,77 @@ int dbox_map_get_file_msgs(struct dbox_map *map, uint32_t file_id, return 0; } +struct dbox_file_size { + uoff_t file_size; + uoff_t ref0_size; +}; + +static void dbox_map_filter_zero_refs(struct dbox_map *map) +{ + ARRAY_TYPE(seq_range) new_ref0_file_ids; + struct hash_table *hash; + struct dbox_file_size *size; + struct seq_range_iter iter; + const struct mail_index_header *hdr; + const struct dbox_mail_index_map_record *rec; + const uint16_t *ref16_p; + const void *data; + uint32_t seq, file_id; + unsigned int i; + bool expunged; + pool_t pool; + + pool = pool_alloconly_create("dbox zero ref count", 8*1024); + hash = hash_table_create(default_pool, pool, 0, NULL, NULL); + + /* count file sizes */ + hdr = mail_index_get_header(map->view); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_index_lookup_ext(map->view, seq, map->map_ext_id, + &data, &expunged); + if (data == NULL || expunged) + continue; + rec = data; + + if (!seq_range_exists(&map->ref0_file_ids, rec->file_id)) + continue; + + /* this file has at least some zero references. count how many + bytes it has in total and how much of it has refcount=0. */ + mail_index_lookup_ext(map->view, seq, map->ref_ext_id, + &data, &expunged); + if (data == NULL || expunged) + continue; + ref16_p = data; + + size = hash_table_lookup(hash, POINTER_CAST(rec->file_id)); + if (size == NULL) { + size = p_new(pool, struct dbox_file_size, 1); + hash_table_insert(hash, POINTER_CAST(rec->file_id), + size); + } + if (*ref16_p == 0) + size->ref0_size += rec->size; + if (size->file_size < rec->offset + rec->size) + size->file_size = rec->offset + rec->size; + } + + /* now drop the files that don't have enough deleted space */ + seq_range_array_iter_init(&iter, &map->ref0_file_ids); i = 0; + p_array_init(&new_ref0_file_ids, pool, + array_count(&map->ref0_file_ids)); + while (seq_range_array_iter_nth(&iter, i++, &file_id)) { + size = hash_table_lookup(hash, POINTER_CAST(file_id)); + if (size->ref0_size*100 / size->file_size >= + map->storage->purge_min_percentage) + seq_range_array_add(&new_ref0_file_ids, 0, file_id); + } + seq_range_array_intersect(&map->ref0_file_ids, &new_ref0_file_ids); + + hash_table_destroy(&hash); + pool_unref(&pool); +} + const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map) { const struct mail_index_header *hdr; @@ -255,6 +327,11 @@ const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map) else i_array_init(&map->ref0_file_ids, 64); + if (map->storage->purge_min_percentage >= 100) { + /* we're never purging anything */ + return &map->ref0_file_ids; + } + if (dbox_map_open(map, FALSE) < 0) { /* some internal error */ return &map->ref0_file_ids; @@ -279,6 +356,9 @@ const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map) rec->file_id); } } + if (map->storage->purge_min_percentage > 0 && + array_count(&map->ref0_file_ids) > 0) + dbox_map_filter_zero_refs(map); return &map->ref0_file_ids; } diff --git a/src/lib-storage/index/dbox/dbox-storage.c b/src/lib-storage/index/dbox/dbox-storage.c index b540988ee3..893fb89448 100644 --- a/src/lib-storage/index/dbox/dbox-storage.c +++ b/src/lib-storage/index/dbox/dbox-storage.c @@ -168,6 +168,11 @@ static int dbox_create(struct mail_storage *_storage, const char *data, storage->max_open_files = (unsigned int)strtoul(value, NULL, 10); else storage->max_open_files = DBOX_DEFAULT_MAX_OPEN_FILES; + value = getenv("DBOX_PURGE_MIN_PERCENTAGE"); + if (value != NULL) + storage->purge_min_percentage = (unsigned int)strtoul(value, NULL, 10); + else + storage->purge_min_percentage = DBOX_DEFAULT_PURGE_MIN_PERCENTAGE; if (storage->max_open_files <= 1) { /* we store file offsets in a 32bit integer. */ diff --git a/src/lib-storage/index/dbox/dbox-storage.h b/src/lib-storage/index/dbox/dbox-storage.h index da0fa134e1..b5f96c0bb9 100644 --- a/src/lib-storage/index/dbox/dbox-storage.h +++ b/src/lib-storage/index/dbox/dbox-storage.h @@ -25,11 +25,12 @@ /* Delete temp files having ctime older than this. */ #define DBOX_TMP_DELETE_SECS (36*60*60) -/* Default rotation settings */ +/* Default dbox settings */ #define DBOX_DEFAULT_ROTATE_SIZE (2*1024*1024) #define DBOX_DEFAULT_ROTATE_MIN_SIZE (1024*16) #define DBOX_DEFAULT_ROTATE_DAYS 0 #define DBOX_DEFAULT_MAX_OPEN_FILES 64 +#define DBOX_DEFAULT_PURGE_MIN_PERCENTAGE 0 /* Flag specifies if the message should be in primary or alternative storage */ #define DBOX_INDEX_FLAG_ALT MAIL_INDEX_MAIL_FLAG_BACKEND @@ -56,6 +57,7 @@ struct dbox_storage { uoff_t rotate_size, rotate_min_size; unsigned int rotate_days; unsigned int max_open_files; + unsigned int purge_min_percentage; ARRAY_DEFINE(open_files, struct dbox_file *); unsigned int sync_rebuild:1; diff --git a/src/master/mail-process.c b/src/master/mail-process.c index e58944706e..e356b30ed6 100644 --- a/src/master/mail-process.c +++ b/src/master/mail-process.c @@ -399,6 +399,8 @@ mail_process_set_environment(struct settings *set, const char *mail, set->dbox_rotate_min_size)); env_put(t_strdup_printf("DBOX_ROTATE_DAYS=%u", set->dbox_rotate_days)); + env_put(t_strdup_printf("DBOX_PURGE_MIN_PERCENTAGE=%u", + set->dbox_purge_min_percentage)); if (*set->mail_plugins != '\0') { env_put(t_strconcat("MAIL_PLUGIN_DIR=", diff --git a/src/master/master-settings-defs.c b/src/master/master-settings-defs.c index 6d7e3bdb96..a8337725d0 100644 --- a/src/master/master-settings-defs.c +++ b/src/master/master-settings-defs.c @@ -98,6 +98,7 @@ static struct setting_def setting_defs[] = { DEF_INT(dbox_rotate_size), DEF_INT(dbox_rotate_min_size), DEF_INT(dbox_rotate_days), + DEF_INT(dbox_purge_min_percentage), DEF_INT(umask), DEF_BOOL(mail_drop_priv_before_exec), diff --git a/src/master/master-settings.c b/src/master/master-settings.c index 39ad71a958..771024e0d4 100644 --- a/src/master/master-settings.c +++ b/src/master/master-settings.c @@ -264,6 +264,7 @@ struct settings default_settings = { MEMBER(dbox_rotate_size) 2048, MEMBER(dbox_rotate_min_size) 16, MEMBER(dbox_rotate_days) 1, + MEMBER(dbox_purge_min_percentage) 0, MEMBER(umask) 0077, MEMBER(mail_drop_priv_before_exec) FALSE, diff --git a/src/master/master-settings.h b/src/master/master-settings.h index 9ffb981b1b..96162a75a9 100644 --- a/src/master/master-settings.h +++ b/src/master/master-settings.h @@ -110,6 +110,7 @@ struct settings { unsigned int dbox_rotate_size; unsigned int dbox_rotate_min_size; unsigned int dbox_rotate_days; + unsigned int dbox_purge_min_percentage; unsigned int umask; bool mail_drop_priv_before_exec;