dbox: Added dbox_purge_min_percentage setting.

--HG--
branch : HEAD

diff --git a/dovecot-example.conf b/dovecot-example.conf
index 6d6e055b6afe8b8d388f166de64089980124e990..f27c444af3e1c81222e2d2853e2d0e31d3c2269f 100644 (file)
--- a/dovecot-example.conf
+++ b/dovecot-example.conf
@@ -526,6 +526,10 @@
 # midnight, so 1 = today, 2 = yesterday, etc. 0 = check disabled.
 #dbox_rotate_days = 0
 
+# Don't purge a dbox file until this many % of it contains expunged messages.
+# 0 purges always, 100 purges never.
+#dbox_purge_min_percentage = 0
+
 ##
 ## IMAP specific settings
 ##

diff --git a/src/lib-storage/index/dbox/dbox-map.c b/src/lib-storage/index/dbox/dbox-map.c
index e4dda0dce976be6f266671873c0bdefccbbe8b21..723e0dcd852f79096751ddd24f2a4869df1d25a1 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-map.c
+++ b/src/lib-storage/index/dbox/dbox-map.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "array.h"
+#include "hash.h"
 #include "ostream.h"
 #include "mkdir-parents.h"
 #include "dbox-storage.h"
@@ -241,6 +242,77 @@ int dbox_map_get_file_msgs(struct dbox_map *map, uint32_t file_id,
        return 0;
 }
 
+struct dbox_file_size {
+       uoff_t file_size;
+       uoff_t ref0_size;
+};
+
+static void dbox_map_filter_zero_refs(struct dbox_map *map)
+{
+       ARRAY_TYPE(seq_range) new_ref0_file_ids;
+       struct hash_table *hash;
+       struct dbox_file_size *size;
+       struct seq_range_iter iter;
+       const struct mail_index_header *hdr;
+       const struct dbox_mail_index_map_record *rec;
+       const uint16_t *ref16_p;
+       const void *data;
+       uint32_t seq, file_id;
+       unsigned int i;
+       bool expunged;
+       pool_t pool;
+
+       pool = pool_alloconly_create("dbox zero ref count", 8*1024);
+       hash = hash_table_create(default_pool, pool, 0, NULL, NULL);
+
+       /* count file sizes */
+       hdr = mail_index_get_header(map->view);
+       for (seq = 1; seq <= hdr->messages_count; seq++) {
+               mail_index_lookup_ext(map->view, seq, map->map_ext_id,
+                                     &data, &expunged);
+               if (data == NULL || expunged)
+                       continue;
+               rec = data;
+
+               if (!seq_range_exists(&map->ref0_file_ids, rec->file_id))
+                       continue;
+
+               /* this file has at least some zero references. count how many
+                  bytes it has in total and how much of it has refcount=0. */
+               mail_index_lookup_ext(map->view, seq, map->ref_ext_id,
+                                     &data, &expunged);
+               if (data == NULL || expunged)
+                       continue;
+               ref16_p = data;
+
+               size = hash_table_lookup(hash, POINTER_CAST(rec->file_id));
+               if (size == NULL) {
+                       size = p_new(pool, struct dbox_file_size, 1);
+                       hash_table_insert(hash, POINTER_CAST(rec->file_id),
+                                         size);
+               }
+               if (*ref16_p == 0)
+                       size->ref0_size += rec->size;
+               if (size->file_size < rec->offset + rec->size)
+                       size->file_size = rec->offset + rec->size;
+       }
+
+       /* now drop the files that don't have enough deleted space */
+       seq_range_array_iter_init(&iter, &map->ref0_file_ids); i = 0;
+       p_array_init(&new_ref0_file_ids, pool, 
+                    array_count(&map->ref0_file_ids));
+       while (seq_range_array_iter_nth(&iter, i++, &file_id)) {
+               size = hash_table_lookup(hash, POINTER_CAST(file_id));
+               if (size->ref0_size*100 / size->file_size >=
+                   map->storage->purge_min_percentage)
+                       seq_range_array_add(&new_ref0_file_ids, 0, file_id);
+       }
+       seq_range_array_intersect(&map->ref0_file_ids, &new_ref0_file_ids);
+
+       hash_table_destroy(&hash);
+       pool_unref(&pool);
+}
+
 const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map)
 {
        const struct mail_index_header *hdr;
@@ -255,6 +327,11 @@ const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map)
        else
                i_array_init(&map->ref0_file_ids, 64);
 
+       if (map->storage->purge_min_percentage >= 100) {
+               /* we're never purging anything */
+               return &map->ref0_file_ids;
+       }
+
        if (dbox_map_open(map, FALSE) < 0) {
                /* some internal error */
                return &map->ref0_file_ids;
@@ -279,6 +356,9 @@ const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map)
                                            rec->file_id);
                }
        }
+       if (map->storage->purge_min_percentage > 0 &&
+           array_count(&map->ref0_file_ids) > 0)
+               dbox_map_filter_zero_refs(map);
        return &map->ref0_file_ids;
 }
 

diff --git a/src/lib-storage/index/dbox/dbox-storage.c b/src/lib-storage/index/dbox/dbox-storage.c
index b540988ee344e893a04054dd921a42ac4bbacd90..893fb894486af2f90f963bde2e4b2da6655ce184 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-storage.c
+++ b/src/lib-storage/index/dbox/dbox-storage.c
@@ -168,6 +168,11 @@ static int dbox_create(struct mail_storage *_storage, const char *data,
                storage->max_open_files = (unsigned int)strtoul(value, NULL, 10);
        else
                storage->max_open_files = DBOX_DEFAULT_MAX_OPEN_FILES;
+       value = getenv("DBOX_PURGE_MIN_PERCENTAGE");
+       if (value != NULL)
+               storage->purge_min_percentage = (unsigned int)strtoul(value, NULL, 10);
+       else
+               storage->purge_min_percentage = DBOX_DEFAULT_PURGE_MIN_PERCENTAGE;
 
        if (storage->max_open_files <= 1) {
                /* we store file offsets in a 32bit integer. */

diff --git a/src/lib-storage/index/dbox/dbox-storage.h b/src/lib-storage/index/dbox/dbox-storage.h
index da0fa134e1848b439d6be4f9af20b3e5e7a2c9d3..b5f96c0bb94a7de157339dd6bc9dc2df9322bd97 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-storage.h
+++ b/src/lib-storage/index/dbox/dbox-storage.h
@@ -25,11 +25,12 @@
 /* Delete temp files having ctime older than this. */
 #define DBOX_TMP_DELETE_SECS (36*60*60)
 
-/* Default rotation settings */
+/* Default dbox settings */
 #define DBOX_DEFAULT_ROTATE_SIZE (2*1024*1024)
 #define DBOX_DEFAULT_ROTATE_MIN_SIZE (1024*16)
 #define DBOX_DEFAULT_ROTATE_DAYS 0
 #define DBOX_DEFAULT_MAX_OPEN_FILES 64
+#define DBOX_DEFAULT_PURGE_MIN_PERCENTAGE 0
 
 /* Flag specifies if the message should be in primary or alternative storage */
 #define DBOX_INDEX_FLAG_ALT MAIL_INDEX_MAIL_FLAG_BACKEND
@@ -56,6 +57,7 @@ struct dbox_storage {
        uoff_t rotate_size, rotate_min_size;
        unsigned int rotate_days;
        unsigned int max_open_files;
+       unsigned int purge_min_percentage;
        ARRAY_DEFINE(open_files, struct dbox_file *);
 
        unsigned int sync_rebuild:1;

diff --git a/src/master/mail-process.c b/src/master/mail-process.c
index e58944706e140a1642e98db87c786926f1d01559..e356b30ed61d3986c32957261239c86fc597bc29 100644 (file)
--- a/src/master/mail-process.c
+++ b/src/master/mail-process.c
@@ -399,6 +399,8 @@ mail_process_set_environment(struct settings *set, const char *mail,
                                set->dbox_rotate_min_size));
        env_put(t_strdup_printf("DBOX_ROTATE_DAYS=%u",
                                set->dbox_rotate_days));
+       env_put(t_strdup_printf("DBOX_PURGE_MIN_PERCENTAGE=%u",
+                               set->dbox_purge_min_percentage));
 
        if (*set->mail_plugins != '\0') {
                env_put(t_strconcat("MAIL_PLUGIN_DIR=",

diff --git a/src/master/master-settings-defs.c b/src/master/master-settings-defs.c
index 6d7e3bdb962ede9746ba0a97362341edc97e798e..a8337725d0879a1a401024e0c172257e49a3515f 100644 (file)
--- a/src/master/master-settings-defs.c
+++ b/src/master/master-settings-defs.c
@@ -98,6 +98,7 @@ static struct setting_def setting_defs[] = {
        DEF_INT(dbox_rotate_size),
        DEF_INT(dbox_rotate_min_size),
        DEF_INT(dbox_rotate_days),
+       DEF_INT(dbox_purge_min_percentage),
        DEF_INT(umask),
        DEF_BOOL(mail_drop_priv_before_exec),
 

diff --git a/src/master/master-settings.c b/src/master/master-settings.c
index 39ad71a958e084c521d3a13951520459d82661b7..771024e0d4bda37b1b4acc313a3088ae906aaaad 100644 (file)
--- a/src/master/master-settings.c
+++ b/src/master/master-settings.c
@@ -264,6 +264,7 @@ struct settings default_settings = {
        MEMBER(dbox_rotate_size) 2048,
        MEMBER(dbox_rotate_min_size) 16,
        MEMBER(dbox_rotate_days) 1,
+       MEMBER(dbox_purge_min_percentage) 0,
        MEMBER(umask) 0077,
        MEMBER(mail_drop_priv_before_exec) FALSE,
 

diff --git a/src/master/master-settings.h b/src/master/master-settings.h
index 9ffb981b1bee90e46cf520cb3f9991a4fdaa8a9b..96162a75a9b77bdba5bf093dfddec8e383955750 100644 (file)
--- a/src/master/master-settings.h
+++ b/src/master/master-settings.h
@@ -110,6 +110,7 @@ struct settings {
        unsigned int dbox_rotate_size;
        unsigned int dbox_rotate_min_size;
        unsigned int dbox_rotate_days;
+       unsigned int dbox_purge_min_percentage;
        unsigned int umask;
        bool mail_drop_priv_before_exec;