]> git.ipfire.org Git - thirdparty/dovecot/core.git/commitdiff
index_mmap_invalidate = yes now invalidate memory maps before accessing
authorTimo Sirainen <tss@iki.fi>
Tue, 15 Jul 2003 18:26:42 +0000 (21:26 +0300)
committerTimo Sirainen <tss@iki.fi>
Tue, 15 Jul 2003 18:26:42 +0000 (21:26 +0300)
them. Setting this on should fix some problems with OpenBSD. It should also
make it possible to use index files over NFS as long as lock daemon is used.
It might be such a good idea however.

--HG--
branch : HEAD

22 files changed:
configure.in
dovecot-example.conf
src/imap/cmd-append.c
src/imap/cmd-copy.c
src/imap/cmd-select.c
src/imap/cmd-status.c
src/imap/common.h
src/imap/main.c
src/lib-index/mail-index-data.c
src/lib-index/mail-index.c
src/lib-index/mail-index.h
src/lib-index/mail-modifylog.c
src/lib-index/mail-tree.c
src/lib-storage/index/index-storage.c
src/lib-storage/index/index-storage.h
src/lib-storage/index/maildir/maildir-storage.c
src/lib-storage/index/mbox/mbox-storage.c
src/lib-storage/mail-storage.h
src/master/mail-process.c
src/master/master-settings.c
src/master/master-settings.h
src/pop3/client.c

index e72d4e700ae2764ec9e2f6aa0b1f0cd154a03cbb..ca9ad24ea98b3b4958c515ca421fe54ad528224e 100644 (file)
@@ -593,6 +593,44 @@ AC_TRY_COMPILE([
   AC_MSG_RESULT(no)
 ])
 
+dnl * If mmap() plays nicely with write()
+AC_MSG_CHECKING([whether we need to use MS_INVALIDATE with mmaps])
+AC_TRY_RUN([
+  #include <stdio.h>
+  #include <sys/types.h>
+  #include <sys/stat.h>
+  #include <unistd.h>
+  #include <fcntl.h>
+  #include <sys/mman.h>
+  int main() {
+    /* return 0 if we're signed */
+    int f = open("conftest.mmap", O_RDWR|O_CREAT|O_TRUNC);
+    void *mem;
+    if (f == -1) {
+      perror("open()");
+      return 1;
+    }
+    write(f, "1", 2);
+    mem = mmap(NULL, 2, PROT_READ|PROT_WRITE, MAP_SHARED, f, 0);
+    if (mem == MAP_FAILED) {
+      perror("mmap()");
+      return 1;
+    }
+    strcpy(mem, "2");
+    msync(mem, 2, MS_SYNC);
+    lseek(f, 0, SEEK_SET);
+    write(f, "3", 2);
+  
+    return strcmp(mem, "3") == 0 ? 0 : 1;
+  }
+],
+  AC_MSG_RESULT(no)
+], [
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(NEED_MS_INVALIDATE,, Define if your mmap() implementation requires use of MS_INVALIDATE to work with write())
+])
+
+
 dnl * Solaris compatible sendfilev()
 AC_CHECK_LIB(sendfile, sendfilev, [
   LIBS="$LIBS -lsendfile"
index a00311255b93f487b10b05387421c51c92bfb0f9..2911a14e001ba6b8bfa8267268b71873a499799c 100644 (file)
 
 # Use mmap() instead of read() to read mail files. read() seems to be a bit
 # faster with my Linux/x86 and it's better with NFS, so that's the default.
+# Note that OpenBSD 3.3 and older don't work right with mail_read_mmaped = yes.
 #mail_read_mmaped = no
 
 # Copy mail to another folders using hard links. This is much faster than
 # for multiple users, as the users could ptrace() each others processes then.
 #mail_drop_priv_before_exec = no
 
+# Call mmap() with MS_INVALIDATE flag each time before accessing message
+# indexes. This is currently required with OpenBSD 3.3 and older versions.
+# It's also required if you insist on saving index files over NFS.
+#index_mmap_invalidate = no
+
 # Set max. process size in megabytes. Most of the memory goes to mmap()ing
 # files, so it shouldn't harm much even if this limit is set pretty high.
 #mail_process_size = 256
index d84759bebe27de9d79cac7b5438204a37c02ffc8..1380f21de938d36c8c60aae7cc4e9a70ecf1b654 100644 (file)
@@ -67,7 +67,8 @@ int cmd_append(struct client *client)
                return TRUE;
 
        box = client->storage->open_mailbox(client->storage,
-                                           mailbox, FALSE, TRUE);
+                                           mailbox, mailbox_open_flags |
+                                           MAILBOX_OPEN_FAST);
        if (box == NULL) {
                client_send_storage_error(client);
                return TRUE;
index 8428a89178f252ba3cc8b90942e20e3a62a8fb27..5245759fde4e10d0fea681cbaa559687174ba924 100644 (file)
@@ -21,7 +21,8 @@ int cmd_copy(struct client *client)
                return TRUE;
 
        destbox = client->storage->open_mailbox(client->storage,
-                                               mailbox, FALSE, TRUE);
+                                               mailbox, mailbox_open_flags |
+                                               MAILBOX_OPEN_FAST);
        if (destbox == NULL) {
                client_send_storage_error(client);
                return TRUE;
index 770db9544ea54795e0f294cde363a4caf4e9f1ee..634e3260daf4a254c32e309ea4b4a9de0d382466 100644 (file)
@@ -7,6 +7,7 @@ int _cmd_select_full(struct client *client, int readonly)
 {
        struct mailbox *box;
        struct mailbox_status status;
+       enum mailbox_open_flags flags;
        const char *mailbox;
 
        /* <mailbox> */
@@ -20,8 +21,10 @@ int _cmd_select_full(struct client *client, int readonly)
                         client_send_untagged_storage_error(client);
        }
 
-       box = client->storage->open_mailbox(client->storage, mailbox,
-                                           readonly, FALSE);
+       flags = mailbox_open_flags;
+       if (readonly)
+               flags |= MAILBOX_OPEN_READONLY;
+       box = client->storage->open_mailbox(client->storage, mailbox, flags);
        if (box == NULL) {
                client_send_storage_error(client);
                return TRUE;
index d53fe1822b9fbf324fbf63b3f7875190cc140ae6..62c04a377d38e73d1b6e760b440d4e422700f5cf 100644 (file)
@@ -65,7 +65,10 @@ static int get_mailbox_status(struct client *client, const char *mailbox,
        } else {
                /* open the mailbox */
                box = client->storage->open_mailbox(client->storage,
-                                                   mailbox, TRUE, TRUE);
+                                                   mailbox,
+                                                   mailbox_open_flags |
+                                                   MAILBOX_OPEN_FAST |
+                                                   MAILBOX_OPEN_READONLY);
                if (box == NULL)
                        return FALSE;
        }
index edcb7979af36f7d24850a399fef8cb967418ea97..667716e1157e61eac8d578b9c4fa393b30bcdc4b 100644 (file)
@@ -17,6 +17,7 @@
 extern struct ioloop *ioloop;
 extern unsigned int max_custom_flag_length, mailbox_check_interval;
 extern unsigned int imap_max_line_length;
+extern enum mailbox_open_flags mailbox_open_flags;
 
 extern string_t *capability_string;
 
index f30ce7716ab622214de0d51543dc37e55b2df0ff..a9be7d701af085acdefd17ea23c7e751b3265b79 100644 (file)
@@ -24,6 +24,7 @@
 struct ioloop *ioloop;
 unsigned int max_custom_flag_length, mailbox_check_interval;
 unsigned int imap_max_line_length;
+enum mailbox_open_flags mailbox_open_flags;
 
 static struct module *modules;
 static char log_prefix[128]; /* syslog() needs this to be permanent */
@@ -147,6 +148,9 @@ static void main_init(void)
        mailbox_check_interval = str == NULL ? 0 :
                (unsigned int)strtoul(str, NULL, 10);
 
+       mailbox_open_flags = getenv("MMAP_INVALIDATE") != NULL ?
+               MAILBOX_OPEN_MMAP_INVALIDATE : 0;
+
        client = client_create(hin, hout, storage);
 
         o_stream_cork(client->output);
index c2f258ba74765f9e03efbfd0406619bb053bfe25..cf30e2a9c5125f913c48a62be98b8e35a62e7891 100644 (file)
@@ -133,6 +133,12 @@ static int mmap_update(struct mail_index_data *data, uoff_t pos, size_t size)
 {
        struct mail_index_data_header *hdr;
 
+       if (data->index->mmap_invalidate && data->mmap_base != NULL) {
+               if (msync(data->mmap_base, data->mmap_used_length,
+                         MS_SYNC | MS_INVALIDATE) < 0)
+                       return index_data_set_syscall_error(data, "msync()");
+       }
+
        if (data->header != NULL &&
            data->header->indexid != data->index->indexid) {
                if (data->header->indexid != 0) {
index 7efe0c8657c028ed13ec1e51d7cdc3994351f96f..4d3c8ff234b82dec9c34ee564213a9a826bcc9e6 100644 (file)
@@ -89,6 +89,15 @@ int mail_index_mmap_update(struct mail_index *index)
        if (index->mmap_base != NULL) {
                index->header = (struct mail_index_header *) index->mmap_base;
 
+               if (index->mmap_invalidate) {
+                       if (msync(index->mmap_base,
+                                 index->mmap_used_length,
+                                 MS_SYNC | MS_INVALIDATE) < 0) {
+                               index_set_syscall_error(index, "msync()");
+                               return FALSE;
+                       }
+               }
+
                /* make sure file size hasn't changed */
                if (index->header->sync_id == index->sync_id) {
                        index->mmap_used_length = index->header->used_file_size;
@@ -99,9 +108,13 @@ int mail_index_mmap_update(struct mail_index *index)
                        return TRUE;
                }
 
-               if (msync(index->mmap_base,
-                         index->mmap_used_length, MS_SYNC) < 0)
-                       return index_set_syscall_error(index, "msync()");
+               if (!index->mmap_invalidate) {
+                       if (msync(index->mmap_base,
+                                 index->mmap_used_length, MS_SYNC) < 0) {
+                               index_set_syscall_error(index, "msync()");
+                               return FALSE;
+                       }
+               }
 
                if (munmap(index->mmap_base, index->mmap_full_length) < 0)
                        return index_set_syscall_error(index, "munmap()");
index e72db06e8ee4007fb45acd8069bce2ba73225a1d..d6864b92ada7592f84deb48c833fae7daf1eee30 100644 (file)
@@ -16,9 +16,11 @@ enum mail_index_open_flags {
        MAIL_INDEX_OPEN_FLAG_UPDATE_RECENT      = 0x02,
        /* Compressing and cache updates are not performed */
        MAIL_INDEX_OPEN_FLAG_FAST               = 0x04,
+       /* Invalidate memory maps before accessing them */
+       MAIL_INDEX_OPEN_FLAG_MMAP_INVALIDATE    = 0x08,
 
        /* internal: we're creating the index */
-       _MAIL_INDEX_OPEN_FLAG_CREATING          = 0x10
+       _MAIL_INDEX_OPEN_FLAG_CREATING          = 0x100
 };
 
 enum mail_index_header_compat {
@@ -433,6 +435,7 @@ struct mail_index {
        enum mail_data_field set_cache_fields;
 
        unsigned int anon_mmap:1;
+       unsigned int mmap_invalidate:1;
        unsigned int opened:1;
        unsigned int rebuilding:1;
        unsigned int mail_read_mmaped:1;
@@ -460,7 +463,7 @@ struct mail_index {
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
-       0, 0, 0, 0, 0
+       0, 0, 0, 0, 0, 0
 #endif
 
 /* defaults - same as above but prefixed with mail_index_. */
index 2738970a4b367943ae706990c9bb8e745eee023b..f198be924415551587b0020136e5ed46f36fe627 100644 (file)
@@ -176,6 +176,12 @@ static int mmap_update(struct modify_log_file *file, int forced)
        struct modify_log_header *hdr;
        unsigned int extra;
 
+       if (file->log->index->mmap_invalidate && file->mmap_base != NULL) {
+               if (msync(file->mmap_base, file->mmap_used_length,
+                         MS_SYNC | MS_INVALIDATE) < 0)
+                       return modifylog_set_syscall_error(file, "msync()");
+       }
+
        if (!forced && file->header != NULL &&
            file->mmap_full_length >= file->header->used_file_size) {
                file->mmap_used_length = file->header->used_file_size;
index 75d173cb45ad39eeee68a322f4419ec7521426f1..e6a1353cc66f32d49712c87cce9c75860a7affe8 100644 (file)
@@ -131,6 +131,12 @@ static int mmap_verify(struct mail_tree *tree)
 
 int _mail_tree_mmap_update(struct mail_tree *tree, int forced)
 {
+       if (tree->index->mmap_invalidate && tree->mmap_base != NULL) {
+               if (msync(tree->mmap_base, tree->mmap_used_length,
+                         MS_SYNC | MS_INVALIDATE) < 0)
+                       return tree_set_syscall_error(tree, "msync()");
+       }
+
        debug_mprotect(tree->mmap_base, tree->mmap_full_length,
                       tree->index);
 
index c8dd2a830a38de544444ebbf0b695485f70ad61c..a3b0049763fe3715c860519e57f79edd3438b2e9 100644 (file)
@@ -299,20 +299,28 @@ int index_storage_lock(struct index_mailbox *ibox,
 struct index_mailbox *
 index_storage_mailbox_init(struct mail_storage *storage, struct mailbox *box,
                           struct mail_index *index, const char *name,
-                          int readonly, int fast)
+                          enum mailbox_open_flags flags)
 {
        struct index_mailbox *ibox;
-       enum mail_index_open_flags flags;
+       enum mail_index_open_flags index_flags;
 
        i_assert(name != NULL);
 
+       index_flags = MAIL_INDEX_OPEN_FLAG_CREATE;
+       if ((flags & MAILBOX_OPEN_FAST) != 0)
+               index_flags |= MAIL_INDEX_OPEN_FLAG_FAST;
+       if ((flags & MAILBOX_OPEN_READONLY) != 0)
+               index_flags |= MAIL_INDEX_OPEN_FLAG_UPDATE_RECENT;
+       if ((flags & MAILBOX_OPEN_MMAP_INVALIDATE) != 0)
+               index_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_INVALIDATE;
+
        do {
                ibox = i_new(struct index_mailbox, 1);
                ibox->box = *box;
 
                ibox->box.storage = storage;
                ibox->box.name = i_strdup(name);
-               ibox->box.readonly = readonly;
+               ibox->box.readonly = (flags & MAILBOX_OPEN_READONLY) != 0;
 
                ibox->index = index;
 
@@ -326,13 +334,7 @@ index_storage_mailbox_init(struct mail_storage *storage, struct mailbox *box,
                        index->never_cache_fields =
                                get_never_cache_fields();
 
-                       flags = MAIL_INDEX_OPEN_FLAG_CREATE;
-                       if (fast)
-                               flags |= MAIL_INDEX_OPEN_FLAG_FAST;
-                       if (!readonly)
-                               flags |= MAIL_INDEX_OPEN_FLAG_UPDATE_RECENT;
-
-                       if (!index->open(index, flags))
+                       if (!index->open(index, index_flags))
                                break;
 
                        if (INDEX_IS_IN_MEMORY(index) &&
index 2ea1e0dcfab337797199539f9048210df06996a6..0a5a4469a9e8382a61fa4825c7d00fd8bc7d9df8 100644 (file)
@@ -52,7 +52,7 @@ void index_storage_deinit(struct mail_storage *storage);
 struct index_mailbox *
 index_storage_mailbox_init(struct mail_storage *storage, struct mailbox *box,
                           struct mail_index *index, const char *name,
-                          int readonly, int fast);
+                          enum mailbox_open_flags flags);
 int index_storage_mailbox_free(struct mailbox *box);
 
 int index_storage_sync_and_lock(struct index_mailbox *ibox,
index ece3c050df01ba73aef204abcc1f8810b0be89f9..0c1292f8d18e2a74a7c7cb08855e31579b83643c 100644 (file)
@@ -321,7 +321,7 @@ static int verify_inbox(struct mail_storage *storage)
 
 static struct mailbox *
 maildir_open(struct mail_storage *storage, const char *name,
-            int readonly, int fast)
+            enum mailbox_open_flags flags)
 {
        struct index_mailbox *ibox;
        struct mail_index *index;
@@ -338,7 +338,7 @@ maildir_open(struct mail_storage *storage, const char *name,
        }
 
        ibox = index_storage_mailbox_init(storage, &maildir_mailbox,
-                                         index, name, readonly, fast);
+                                         index, name, flags);
        if (ibox != NULL)
                ibox->expunge_locked = maildir_expunge_locked;
        return (struct mailbox *) ibox;
@@ -359,7 +359,7 @@ static const char *inbox_fix_case(struct mail_storage *storage,
 
 static struct mailbox *
 maildir_open_mailbox(struct mail_storage *storage,
-                    const char *name, int readonly, int fast)
+                    const char *name, enum mailbox_open_flags flags)
 {
        const char *path;
        struct stat st;
@@ -370,7 +370,7 @@ maildir_open_mailbox(struct mail_storage *storage,
        if (strcmp(name, "INBOX") == 0) {
                if (!verify_inbox(storage))
                        return NULL;
-               return maildir_open(storage, "INBOX", readonly, fast);
+               return maildir_open(storage, "INBOX", flags);
        }
 
        if (!maildir_is_valid_existing_name(name)) {
@@ -386,7 +386,7 @@ maildir_open_mailbox(struct mail_storage *storage,
                    !create_control_dir(storage, name))
                        return FALSE;
 
-               return maildir_open(storage, name, readonly, fast);
+               return maildir_open(storage, name, flags);
        } else if (errno == ENOENT) {
                mail_storage_set_error(storage, "Mailbox doesn't exist: %s",
                                       name);
index af6648d53a6e4e7207b3f77d6e0f94c73fddb5fc..92bd9d326b8f49e4887236ea39f8487b1c6f76b0 100644 (file)
@@ -320,7 +320,7 @@ static const char *mbox_get_path(struct mail_storage *storage, const char *name)
 }
 
 static struct mailbox *mbox_open(struct mail_storage *storage, const char *name,
-                                int readonly, int fast)
+                                enum mailbox_open_flags flags)
 {
        struct index_mailbox *ibox;
        struct mail_index *index;
@@ -347,7 +347,7 @@ static struct mailbox *mbox_open(struct mail_storage *storage, const char *name,
        }
 
        ibox = index_storage_mailbox_init(storage, &mbox_mailbox, index,
-                                         name, readonly, fast);
+                                         name, flags);
        if (ibox != NULL)
                ibox->expunge_locked = mbox_expunge_locked;
        return (struct mailbox *) ibox;
@@ -355,7 +355,7 @@ static struct mailbox *mbox_open(struct mail_storage *storage, const char *name,
 
 static struct mailbox *
 mbox_open_mailbox(struct mail_storage *storage,
-                 const char *name, int readonly, int fast)
+                 const char *name, enum mailbox_open_flags flags)
 {
        const char *path;
        struct stat st;
@@ -367,7 +367,7 @@ mbox_open_mailbox(struct mail_storage *storage,
                /* make sure inbox exists */
                if (!verify_inbox(storage))
                        return FALSE;
-               return mbox_open(storage, "INBOX", readonly, fast);
+               return mbox_open(storage, "INBOX", flags);
        }
 
        if (!mbox_is_valid_existing_name(name)) {
@@ -387,7 +387,7 @@ mbox_open_mailbox(struct mail_storage *storage,
                if (!create_mbox_index_dirs(storage, name))
                        return NULL;
 
-               return mbox_open(storage, name, readonly, fast);
+               return mbox_open(storage, name, flags);
        }
 
        if (ENOTFOUND(errno)) {
index 08f88a73c964302bdfb745786f2ba2e74a9c2d6f..757605cc52966d1c4ec8d51b4829bb0a076220eb 100644 (file)
@@ -5,6 +5,12 @@ struct message_size;
 
 #include "imap-util.h"
 
+enum mailbox_open_flags {
+       MAILBOX_OPEN_READONLY           = 0x01,
+       MAILBOX_OPEN_FAST               = 0x02,
+       MAILBOX_OPEN_MMAP_INVALIDATE    = 0x04
+};
+
 enum mailbox_list_flags {
        MAILBOX_LIST_SUBSCRIBED = 0x01,
        MAILBOX_LIST_FAST_FLAGS = 0x02,
@@ -148,7 +154,7 @@ struct mail_storage {
           with possibly different readonly-state. */
        struct mailbox *(*open_mailbox)(struct mail_storage *storage,
                                        const char *name,
-                                       int readonly, int fast);
+                                       enum mailbox_open_flags flags);
 
        /* name is allowed to contain multiple new hierarchy levels.
           If only_hierarchy is TRUE, the mailbox itself isn't created, just
index b4e5f4b4bab391eed49273a2770ecbaeffe245fc..a57f207e2369a79eca3b5223dea4838327b5008b 100644 (file)
@@ -214,6 +214,8 @@ int create_mail_process(struct login_group *group, int socket,
                env_put("MAILDIR_CHECK_CONTENT_CHANGES=1");
        if (set->mail_full_filesystem_access)
                env_put("FULL_FILESYSTEM_ACCESS=1");
+       if (set->index_mmap_invalidate)
+               env_put("MMAP_INVALIDATE=1");
        (void)umask(set->umask);
 
        env_put(t_strconcat("MBOX_LOCKS=", set->mbox_locks, NULL));
index eccf2a7a80a1399de5ef0fe426d822b5cee9b148..59f634dfed5894aa29782ddd5f66d50ad85f63f1 100644 (file)
@@ -92,6 +92,7 @@ static struct setting_def setting_defs[] = {
        DEF(SET_INT, mbox_dotlock_change_timeout),
        DEF(SET_INT, umask),
        DEF(SET_BOOL, mail_drop_priv_before_exec),
+       DEF(SET_BOOL, index_mmap_invalidate),
 
        DEF(SET_STR, mail_executable),
        DEF(SET_INT, mail_process_size),
@@ -193,6 +194,11 @@ struct settings default_settings = {
        MEMBER(mbox_dotlock_change_timeout) 30,
        MEMBER(umask) 0077,
        MEMBER(mail_drop_priv_before_exec) FALSE,
+#ifdef NEED_MS_INVALIDATE
+       MEMBER(index_mmap_invalidate) TRUE,
+#else
+       MEMBER(index_mmap_invalidate) FALSE,
+#endif
 
        MEMBER(mail_executable) PKG_LIBEXECDIR"/imap",
        MEMBER(mail_process_size) 256,
index 76583b2db6123bca48928bdcce03d50c67b8fb4c..64eb416470a11acf879542984d8fa82dd454e0c2 100644 (file)
@@ -79,6 +79,7 @@ struct settings {
        unsigned int mbox_dotlock_change_timeout;
        unsigned int umask;
        int mail_drop_priv_before_exec;
+       int index_mmap_invalidate;
 
        const char *mail_executable;
        unsigned int mail_process_size;
index 339b1f5a23aeb2270584f2a182f0acd2c835df2d..91968d3434b4e75ce3947b7a566cb76d0d2f4774 100644 (file)
@@ -106,6 +106,7 @@ static int init_mailbox(struct client *client)
 struct client *client_create(int hin, int hout, struct mail_storage *storage)
 {
        struct client *client;
+       enum mailbox_open_flags flags;
 
        client = i_new(struct client, 1);
        client->input = i_stream_create_file(hin, default_pool,
@@ -122,7 +123,9 @@ struct client *client_create(int hin, int hout, struct mail_storage *storage)
 
        storage->set_callbacks(storage, &mail_storage_callbacks, client);
 
-       client->mailbox = storage->open_mailbox(storage, "INBOX", FALSE, FALSE);
+       flags = getenv("MMAP_INVALIDATE") != NULL ?
+               MAILBOX_OPEN_MMAP_INVALIDATE : 0;
+       client->mailbox = storage->open_mailbox(storage, "INBOX", flags);
        if (client->mailbox == NULL) {
                client_send_line(client, "-ERR No INBOX for user.");
                return NULL;