lib-storage: Add mail_location=..:ITERINDEX

This changes mailbox list iteration to work using INDEX directory instead of
the normal mail directory. This can be helpful when the indexes are stored
on a faster storage.

diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c
index 9c4e7195b69183b2ccd2b1987b99b5bb3680a711..f3fc07f7295d6c6056a6f46837e3cb9c6e1e3293 100644 (file)
--- a/src/lib-storage/index/index-storage.c
+++ b/src/lib-storage/index/index-storage.c
@@ -177,7 +177,7 @@ int index_storage_mailbox_exists_full(struct mailbox *box, const char *subdir,
 {
        struct stat st;
        enum mail_error error;
-       const char *path, *path2;
+       const char *path, *path2, *index_path;
        int ret;
 
        /* see if it's selectable */
@@ -194,6 +194,22 @@ int index_storage_mailbox_exists_full(struct mailbox *box, const char *subdir,
                *existence_r = MAILBOX_EXISTENCE_NONE;
                return 0;
        }
+
+       ret = (subdir != NULL || !box->list->set.iter_from_index_dir) ? 0 :
+               mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &index_path);
+       if (ret > 0 && strcmp(path, index_path) != 0) {
+               /* index directory is different - prefer looking it up first
+                  since it might be on a faster storage. since the directory
+                  itself exists also for \NoSelect mailboxes, we'll need to
+                  check the dovecot.index.log existence. */
+               index_path = t_strconcat(index_path, "/", box->index_prefix,
+                                        ".log", NULL);
+               if (stat(index_path, &st) == 0) {
+                       *existence_r = MAILBOX_EXISTENCE_SELECT;
+                       return 0;
+               }
+       }
+
        if (subdir != NULL)
                path = t_strconcat(path, "/", subdir, NULL);
        if (stat(path, &st) == 0) {
@@ -580,6 +596,21 @@ int index_storage_mailbox_create(struct mailbox *box, bool directory)
 
        if ((ret = mailbox_mkdir(box, path, type)) < 0)
                return -1;
+       if (box->list->set.iter_from_index_dir) {
+               /* need to also create the directory to index path or
+                  iteration won't find it. */
+               int ret2;
+
+               if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path) <= 0)
+                       i_unreached();
+               if ((ret2 = mailbox_mkdir(box, path, type)) < 0)
+                       return -1;
+               if (ret == 0 && ret2 > 0) {
+                       /* finish partial creation: existed in mail directory,
+                          but not in index directory. */
+                       ret = 1;
+               }
+       }
        mailbox_refresh_permissions(box);
        if (ret == 0) {
                /* directory already exists */

diff --git a/src/lib-storage/list/mailbox-list-fs-flags.c b/src/lib-storage/list/mailbox-list-fs-flags.c
index b21032742a1d8c9d52996d203142f5a85c26fb31..3913b4cbe964a8d6e75d23cdfcfbb5161d9ae587 100644 (file)
--- a/src/lib-storage/list/mailbox-list-fs-flags.c
+++ b/src/lib-storage/list/mailbox-list-fs-flags.c
@@ -118,7 +118,7 @@ int fs_list_get_mailbox_flags(struct mailbox_list *list,
 
        *flags_r = 0;
 
-       if (*list->set.maildir_name != '\0') {
+       if (*list->set.maildir_name != '\0' && !list->set.iter_from_index_dir) {
                /* maildir_name is set: This is the simple case that works for
                   all mail storage formats, because the only thing that
                   matters for existence or child checks is whether the
@@ -130,9 +130,14 @@ int fs_list_get_mailbox_flags(struct mailbox_list *list,
        }
        /* maildir_name is not set: Now we (may) need to use storage-specific
           code to determine whether the mailbox is selectable or if it has
-          children. */
+          children.
 
-       if (list->v.is_internal_name != NULL &&
+          We're here also when iterating from index directory, because even
+          though maildir_name is set, it's not used for index directory.
+       */
+
+       if (!list->set.iter_from_index_dir &&
+           list->v.is_internal_name != NULL &&
            list->v.is_internal_name(list, fname)) {
                /* skip internal dirs. For example Maildir's cur/new/tmp */
                *flags_r |= MAILBOX_NOSELECT;
@@ -219,7 +224,7 @@ int fs_list_get_mailbox_flags(struct mailbox_list *list,
                return 1;
        }
 
-       if (list->v.is_internal_name == NULL) {
+       if (list->v.is_internal_name == NULL || list->set.iter_from_index_dir) {
                /* This mailbox format doesn't use any special directories
                   (e.g. Maildir's cur/new/tmp). In that case we can look at
                   the directory's link count to determine whether there are

diff --git a/src/lib-storage/list/mailbox-list-fs-iter.c b/src/lib-storage/list/mailbox-list-fs-iter.c
index 51fad9b5854ac4fe276f7fc8e5e452ae0153de69..0c8d3384a3b897c4cd27d2ffc7511928e9b37fcd 100644 (file)
--- a/src/lib-storage/list/mailbox-list-fs-iter.c
+++ b/src/lib-storage/list/mailbox-list-fs-iter.c
@@ -245,10 +245,18 @@ fs_list_get_storage_path(struct fs_list_iterate_context *ctx,
        }
        if (*path != '/') {
                /* non-absolute path. add the mailbox root dir as prefix. */
-               if (!mailbox_list_get_root_path(ctx->ctx.list,
-                                               MAILBOX_LIST_PATH_TYPE_MAILBOX,
-                                               &root))
+               enum mailbox_list_path_type type =
+                       ctx->ctx.list->set.iter_from_index_dir ?
+                       MAILBOX_LIST_PATH_TYPE_INDEX :
+                       MAILBOX_LIST_PATH_TYPE_MAILBOX;
+               if (!mailbox_list_get_root_path(ctx->ctx.list, type, &root))
                        return FALSE;
+               if (ctx->ctx.list->set.iter_from_index_dir &&
+                   ctx->ctx.list->set.mailbox_dir_name[0] != '\0') {
+                       /* append "mailboxes/" to the index root */
+                       root = t_strconcat(root, "/",
+                               ctx->ctx.list->set.mailbox_dir_name, NULL);
+               }
                path = *path == '\0' ? root :
                        t_strconcat(root, "/", path, NULL);
        }

diff --git a/src/lib-storage/list/mailbox-list-fs.c b/src/lib-storage/list/mailbox-list-fs.c
index 6ab2b052a3df669ece2c2a2f8cda7a3b134589c5..d7784d10ecfd9a1cfbbb309f46d81ae619dd3a2f 100644 (file)
--- a/src/lib-storage/list/mailbox-list-fs.c
+++ b/src/lib-storage/list/mailbox-list-fs.c
@@ -287,12 +287,32 @@ static int fs_list_delete_dir(struct mailbox_list *list, const char *name)
 {
        const char *path, *child_name, *child_path, *p;
        char sep;
+       int ret;
 
        if (mailbox_list_get_path(list, name, MAILBOX_LIST_PATH_TYPE_DIR,
                                  &path) <= 0)
                i_unreached();
-       if (fs_list_rmdir(list, name, path) == 0)
-               return 0;
+       ret = fs_list_rmdir(list, name, path);
+       if (!list->set.iter_from_index_dir) {
+               /* it should exist only in the mail directory */
+               if (ret == 0)
+                       return 0;
+       } else if (ret == 0 || errno == ENOENT) {
+               /* the primary list location is the index directory, but it
+                  exists in both index and mail directories. */
+               if (mailbox_list_get_path(list, name, MAILBOX_LIST_PATH_TYPE_INDEX,
+                                         &path) <= 0)
+                       i_unreached();
+               if (fs_list_rmdir(list, name, path) == 0)
+                       return 0;
+               if (ret == 0 && errno == ENOENT) {
+                       /* partial existence: exists in _DIR, but not in
+                          _INDEX. return success anyway. */
+                       return 0;
+               }
+               /* a) both directories didn't exist
+                  b) index directory couldn't be rmdir()ed for some reason */
+       }
 
        if (errno == ENOENT || errno == ENOTDIR) {
                mailbox_list_set_error(list, MAIL_ERROR_NOTFOUND,

diff --git a/src/lib-storage/list/mailbox-list-maildir-iter.c b/src/lib-storage/list/mailbox-list-maildir-iter.c
index 83cdddeaec3d05f0d986be69f86d98b5a46082c2..64a63a566e1bdfbe365d75f78e3d162082de876e 100644 (file)
--- a/src/lib-storage/list/mailbox-list-maildir-iter.c
+++ b/src/lib-storage/list/mailbox-list-maildir-iter.c
@@ -441,7 +441,10 @@ maildir_list_iter_init(struct mailbox_list *_list, const char *const *patterns,
        ctx->prefix_char = strcmp(_list->name, MAILBOX_LIST_NAME_IMAPDIR) == 0 ?
                '\0' : list->sep;
 
-       ctx->dir = _list->set.root_dir;
+       if (_list->set.iter_from_index_dir)
+               ctx->dir = _list->set.index_dir;
+       else
+               ctx->dir = _list->set.root_dir;
 
        if ((flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) != 0) {
                /* Listing only subscribed mailboxes.

diff --git a/src/lib-storage/mailbox-list.c b/src/lib-storage/mailbox-list.c
index bde349014884b72dd4cc2a492ca8738b6521358d..c46f1977a3fcdce4963f87d7fdfcfe35e6d1ae6c 100644 (file)
--- a/src/lib-storage/mailbox-list.c
+++ b/src/lib-storage/mailbox-list.c
@@ -175,6 +175,7 @@ int mailbox_list_create(const char *driver, struct mail_namespace *ns,
        list->set.volatile_dir = p_strdup(list->pool, set->volatile_dir);
        list->set.index_control_use_maildir_name =
                set->index_control_use_maildir_name;
+       list->set.iter_from_index_dir = set->iter_from_index_dir;
 
        if (*set->mailbox_dir_name == '\0')
                list->set.mailbox_dir_name = "";
@@ -345,6 +346,9 @@ mailbox_list_settings_parse_full(struct mail_user *user, const char *data,
                else if (strcmp(key, "FULLDIRNAME") == 0) {
                        set_r->index_control_use_maildir_name = TRUE;
                        dest = &set_r->maildir_name;
+               } else if (strcmp(key, "ITERINDEX") == 0) {
+                       set_r->iter_from_index_dir = TRUE;
+                       continue;
                } else {
                        *error_r = t_strdup_printf("Unknown setting: %s", key);
                        return -1;
@@ -357,6 +361,11 @@ mailbox_list_settings_parse_full(struct mail_user *user, const char *data,
 
        if (set_r->index_dir != NULL && strcmp(set_r->index_dir, "MEMORY") == 0)
                set_r->index_dir = "";
+       if (set_r->iter_from_index_dir &&
+           (set_r->index_dir == NULL || set_r->index_dir[0] == '\0')) {
+               *error_r = "ITERINDEX requires INDEX to be explicitly set";
+               return -1;
+       }
        return 0;
 }
 
@@ -1489,9 +1498,15 @@ int mailbox_list_mailbox(struct mailbox_list *list, const char *name,
                return mailbox_list_iter_deinit(&iter);
        }
 
-       rootdir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_MAILBOX);
-       if (mailbox_list_get_path(list, name, MAILBOX_LIST_PATH_TYPE_DIR, &path) <= 0)
-               i_unreached();
+       if (!list->set.iter_from_index_dir) {
+               rootdir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_MAILBOX);
+               if (mailbox_list_get_path(list, name, MAILBOX_LIST_PATH_TYPE_DIR, &path) <= 0)
+                       i_unreached();
+       } else {
+               rootdir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_INDEX);
+               if (mailbox_list_get_path(list, name, MAILBOX_LIST_PATH_TYPE_INDEX, &path) <= 0)
+                       i_unreached();
+       }
 
        fname = strrchr(path, '/');
        if (fname == NULL) {

diff --git a/src/lib-storage/mailbox-list.h b/src/lib-storage/mailbox-list.h
index 03ef584b53aa9d9365bbac7b0081c1ba0217d7d9..6a2595edb8e12a50f5401492151141960d7f6906 100644 (file)
--- a/src/lib-storage/mailbox-list.h
+++ b/src/lib-storage/mailbox-list.h
@@ -139,6 +139,11 @@ struct mailbox_list_settings {
           have been the default since the beginning, but for backwards
           compatibility it had to be made an option. */
        bool index_control_use_maildir_name;
+       /* Perform mailbox iteration using the index directory instead of the
+          mail root directory. This can be helpful if the indexes are on a
+          faster storage. This could perhaps be made the default at some point,
+          but for now since it's less tested it's optional. */
+       bool iter_from_index_dir;
 };
 
 struct mailbox_permissions {