dbox: Store message sizes to map index and use them. Some other save optimizations.

--HG--
branch : HEAD

diff --git a/src/lib-storage/index/dbox/dbox-file.c b/src/lib-storage/index/dbox/dbox-file.c
index 66df4261c25568c2c8802092ca82aba402b393f7..b64699614ecc858a5d24c76dc9480d6caf8a43de 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-file.c
+++ b/src/lib-storage/index/dbox/dbox-file.c
@@ -658,32 +658,45 @@ int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset,
 }
 
 static int
-dbox_file_seek_append_pos(struct dbox_file *file, uoff_t last_msg_offset)
+dbox_file_seek_append_pos(struct dbox_file *file,
+                         uoff_t last_msg_offset, uoff_t last_msg_size)
 {
        uoff_t offset, size;
+       struct stat st;
        bool last;
        int ret;
 
-       if ((ret = dbox_file_read_header(file)) <= 0)
-               return ret;
-
        if (file->file_version != DBOX_VERSION ||
            file->msg_header_size != sizeof(struct dbox_message_header)) {
                /* created by an incompatible version, can't append */
                return 0;
        }
+       if (last_msg_size > 0) {
+               /* if last_msg_offset + last_msg_size points to EOF,
+                  we can safely use it without reading the last message. */
+               if (fstat(file->fd, &st) < 0) {
+                       dbox_file_set_syscall_error(file, "fstat()");
+                       return -1;
+               }
+       } else {
+               st.st_size = -1;
+       }
 
        offset = last_msg_offset;
-       ret = dbox_file_seek_next(file, &offset, &size, &last);
-       if (ret <= 0)
-               return ret;
-       if (!last) {
-               /* not end of file? previous write probably crashed. */
-               if (ftruncate(file->fd, offset) < 0) {
-                       dbox_file_set_syscall_error(file, "ftruncate()");
-                       return -1;
+       if (st.st_size == (off_t)(last_msg_offset + last_msg_size)) {
+               offset += last_msg_size;
+       } else {
+               ret = dbox_file_seek_next(file, &offset, &size, &last);
+               if (ret <= 0)
+                       return ret;
+               if (!last) {
+                       /* not end of file? previous write probably crashed. */
+                       if (ftruncate(file->fd, offset) < 0) {
+                               dbox_file_set_syscall_error(file, "ftruncate()");
+                               return -1;
+                       }
+                       i_stream_sync(file->input);
                }
-               i_stream_sync(file->input);
        }
 
        file->output = o_stream_create_fd_file(file->fd, 0, FALSE);
@@ -692,7 +705,7 @@ dbox_file_seek_append_pos(struct dbox_file *file, uoff_t last_msg_offset)
 }
 
 int dbox_file_get_append_stream(struct dbox_file *file, uoff_t last_msg_offset,
-                               struct ostream **stream_r)
+                               uoff_t last_msg_size, struct ostream **stream_r)
 {
        bool deleted;
        int ret;
@@ -711,7 +724,8 @@ int dbox_file_get_append_stream(struct dbox_file *file, uoff_t last_msg_offset,
        if (file->output == NULL) {
                i_assert(file->lock != NULL || last_msg_offset == 0);
 
-               ret = dbox_file_seek_append_pos(file, last_msg_offset);
+               ret = dbox_file_seek_append_pos(file, last_msg_offset,
+                                               last_msg_size);
                if (ret <= 0)
                        return ret;
        }

diff --git a/src/lib-storage/index/dbox/dbox-file.h b/src/lib-storage/index/dbox/dbox-file.h
index 5a3e0e33269abd23f870a292599cfa0592d575b1..d29d8e541f9b5f6a866079bac206bd701f05d1ff 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-file.h
+++ b/src/lib-storage/index/dbox/dbox-file.h
@@ -158,6 +158,7 @@ bool dbox_file_can_append(struct dbox_file *file, uoff_t mail_size);
    1 if ok, 0 if file can't be appended to (old file version or corruption)
    or -1 if error. */
 int dbox_file_get_append_stream(struct dbox_file *file, uoff_t last_msg_offset,
+                               uoff_t last_msg_size,
                                struct ostream **stream_r);
 /* Returns the next offset for append a message. dbox_file_get_append_stream()
    must have been called for this file already at least once. */

diff --git a/src/lib-storage/index/dbox/dbox-map.c b/src/lib-storage/index/dbox/dbox-map.c
index 013fef55cd06137b68c3cebdc270affce9c1731b..1f2d36af053716b3c76802974dc7dd027fd2fa8e 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-map.c
+++ b/src/lib-storage/index/dbox/dbox-map.c
@@ -16,6 +16,7 @@ struct dbox_mail_index_map_header {
 struct dbox_mail_index_map_record {
        uint32_t file_id;
        uint32_t offset;
+       uint32_t size;
 };
 
 struct dbox_map {
@@ -28,7 +29,7 @@ struct dbox_map {
 
 struct dbox_map_append {
        struct dbox_file *file;
-       uoff_t offset;
+       uoff_t offset, size;
 };
 
 struct dbox_map_append_context {
@@ -118,7 +119,8 @@ static int dbox_map_refresh(struct dbox_map *map)
 }
 
 static int dbox_map_lookup_seq(struct dbox_map *map, uint32_t seq,
-                              uint32_t *file_id_r, uoff_t *offset_r)
+                              uint32_t *file_id_r, uoff_t *offset_r,
+                              uoff_t *size_r)
 {
        const struct dbox_mail_index_map_record *rec;
        const void *data;
@@ -138,6 +140,7 @@ static int dbox_map_lookup_seq(struct dbox_map *map, uint32_t seq,
 
        *file_id_r = rec->file_id;
        *offset_r = rec->offset;
+       *size_r = rec->size;
        return 0;
 }
 
@@ -145,6 +148,7 @@ int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
                    uint32_t *file_id_r, uoff_t *offset_r)
 {
        uint32_t seq;
+       uoff_t size;
        int ret;
 
        if ((ret = dbox_map_open(map, FALSE)) <= 0) {
@@ -160,7 +164,7 @@ int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
                        return 0;
        }
 
-       if (dbox_map_lookup_seq(map, seq, file_id_r, offset_r) < 0)
+       if (dbox_map_lookup_seq(map, seq, file_id_r, offset_r, &size) < 0)
                return 0;
        return 1;
 }
@@ -209,35 +213,36 @@ static time_t day_begin_stamp(unsigned int days)
 
 static bool
 dbox_map_file_try_append(struct dbox_map_append_context *ctx,
-                        uint32_t file_id, time_t stamp,
-                        uoff_t mail_size, uoff_t last_msg_offset,
+                        uint32_t file_id, time_t stamp, uoff_t mail_size,
+                        uoff_t last_msg_offset, uoff_t last_msg_size,
                         struct dbox_file **file_r, struct ostream **output_r,
                         bool *retry_later_r)
 {
        struct dbox_map *map = ctx->map;
+       struct dbox_storage *storage = map->storage;
        const struct mail_index_header *hdr;
        struct dbox_file *file;
        struct stat st;
        uint32_t seq, tmp_file_id;
-       uoff_t tmp_offset;
+       uoff_t tmp_offset, tmp_size;
        bool deleted, file_too_old = FALSE;
        int ret;
 
        *file_r = NULL;
        *retry_later_r = FALSE;
 
-       file = dbox_file_init_multi(map->storage, file_id);
+       file = dbox_file_init_multi(storage, file_id);
        if (dbox_file_open_or_create(file, &deleted) <= 0)
                return TRUE;
        if (deleted)
                return TRUE;
 
        if (fstat(file->fd, &st) < 0) {
-               mail_storage_set_critical(&map->storage->storage,
+               mail_storage_set_critical(&storage->storage,
                        "fstat(%s) failed: %m", file->current_path);
-       } else if (I_MIN(st.st_mtime, st.st_ctime) < stamp)
+       } else if (file->create_time < stamp)
                file_too_old = TRUE;
-       else if (st.st_size + mail_size > map->storage->rotate_size) {
+       else if (st.st_size + mail_size > storage->rotate_size) {
                /* file too large */
        } else if ((ret = dbox_file_try_lock(file)) <= 0) {
                /* locking failed */
@@ -248,18 +253,20 @@ dbox_map_file_try_append(struct dbox_map_append_context *ctx,
                hdr = mail_index_get_header(map->view);
                seq = ctx->orig_msg_count + 1;
                for (; seq <= hdr->messages_count; seq++) {
-                       if (dbox_map_lookup_seq(map, seq,
-                                               &tmp_file_id, &tmp_offset) < 0)
+                       if (dbox_map_lookup_seq(map, seq, &tmp_file_id,
+                                               &tmp_offset, &tmp_size) < 0)
                                break;
                        if (tmp_file_id == file->file_id) {
                                i_assert(last_msg_offset < tmp_offset);
                                last_msg_offset = tmp_offset;
+                               last_msg_size = tmp_size;
                        }
                }
 
                if (seq > hdr->messages_count &&
+                   last_msg_offset + last_msg_size > storage->rotate_size &&
                    dbox_file_get_append_stream(file, last_msg_offset,
-                                               output_r) > 0) {
+                                               last_msg_size, output_r) > 0) {
                        /* success */
                        *file_r = file;
                        return TRUE;
@@ -298,7 +305,7 @@ dbox_map_find_appendable_file(struct dbox_map_append_context *ctx,
        const struct mail_index_header *hdr;
        unsigned int i, count, backwards_lookup_count;
        uint32_t seq, file_id, min_seen_file_id;
-       uoff_t offset, append_offset;
+       uoff_t offset, append_offset, size;
        time_t stamp;
        bool retry_later, updated_retry_seq_min;
 
@@ -310,9 +317,14 @@ dbox_map_find_appendable_file(struct dbox_map_append_context *ctx,
        /* first try to use files already used in this append */
        files = array_get(&ctx->files, &count);
        for (i = count; i > ctx->files_nonappendable_count; i--) {
+               if (files[i-1]->output == NULL) {
+                       /* we already decided we can't append to this */
+                       continue;
+               }
+
                append_offset = dbox_file_get_next_append_offset(files[i-1]);
                if (append_offset + mail_size <= map->storage->rotate_size &&
-                   dbox_file_get_append_stream(files[i-1], append_offset,
+                   dbox_file_get_append_stream(files[i-1], 0, 0,
                                                output_r) > 0) {
                        *file_r = files[i-1];
                        *existing_r = TRUE;
@@ -343,16 +355,20 @@ dbox_map_find_appendable_file(struct dbox_map_append_context *ctx,
                        seq = ctx->retry_seq_min + 1;
                        continue;
                }
-               if (dbox_map_lookup_seq(map, seq, &file_id, &offset) < 0)
+               if (dbox_map_lookup_seq(map, seq, &file_id, &offset, &size) < 0)
                        return -1;
                if (file_id >= min_seen_file_id)
                        continue;
                min_seen_file_id = file_id;
 
-               /* first lookup: this doesn't really tell us if we're going
-                  to exceed the rotate size, it just potentially reduces
-                  how many files we have to open. */
-               if (offset + mail_size >= map->storage->rotate_size)
+               if (++backwards_lookup_count > MAX_BACKWARDS_LOOKUPS) {
+                       /* we've wasted enough time here */
+                       break;
+               }
+
+               /* first lookup: this should be enough usually, but we can't
+                  be sure until after locking */
+               if (offset + size + mail_size >= map->storage->rotate_size)
                        continue;
 
                if (dbox_map_is_appending(ctx, file_id)) {
@@ -360,14 +376,9 @@ dbox_map_find_appendable_file(struct dbox_map_append_context *ctx,
                        continue;
                }
 
-               if (++backwards_lookup_count > MAX_BACKWARDS_LOOKUPS) {
-                       /* we've wasted enough time here */
-                       break;
-               }
-
-               if (!dbox_map_file_try_append(ctx, file_id, stamp,
-                                             mail_size, offset,
-                                             file_r, output_r, &retry_later)) {
+               if (!dbox_map_file_try_append(ctx, file_id, stamp, mail_size,
+                                             offset, size, file_r, output_r,
+                                             &retry_later)) {
                        /* file is too old. the rest of the files are too. */
                        break;
                }
@@ -410,7 +421,7 @@ int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size,
                file = ctx->map->storage->rotate_size == 0 ?
                        dbox_file_init_single(ctx->mbox, 0) :
                        dbox_file_init_multi(ctx->map->storage, 0);
-               ret = dbox_file_get_append_stream(file, 0, output_r);
+               ret = dbox_file_get_append_stream(file, 0, 0, output_r);
                if (ret <= 0) {
                        i_assert(ret < 0);
                        (void)unlink(file->current_path);
@@ -423,6 +434,7 @@ int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size,
                append = array_append_space(&ctx->appends);
                append->file = file;
                append->offset = (*output_r)->offset;
+               append->size = (uint32_t)-1;
        }
        if (!existing) {
                i_assert(file->output != NULL);
@@ -432,6 +444,17 @@ int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size,
        return 0;
 }
 
+void dbox_map_append_finish_mail(struct dbox_map_append_context *ctx)
+{
+       struct dbox_map_append *appends;
+       unsigned int count;
+
+       appends = array_get_modifiable(&ctx->appends, &count);
+       i_assert(count > 0 && appends[count-1].size == (uint32_t)-1);
+       appends[count-1].size = appends[count-1].file->output->offset -
+               appends[count-1].offset;
+}
+
 static int
 dbox_map_get_next_file_id(struct dbox_map *map, struct mail_index_view *view,
                          uint32_t *file_id_r)
@@ -534,8 +557,12 @@ int dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx,
        ref16 = 1;
        appends = array_get(&ctx->appends, &count);
        for (i = 0; i < count; i++) {
+               i_assert(appends[i].offset <= (uint32_t)-1);
+               i_assert(appends[i].size <= (uint32_t)-1);
+
                rec.file_id = appends[i].file->file_id;
                rec.offset = appends[i].offset;
+               rec.size = appends[i].size;
 
                mail_index_append(trans, 0, &seq);
                mail_index_update_ext(trans, seq, ctx->map->map_ext_id,

diff --git a/src/lib-storage/index/dbox/dbox-map.h b/src/lib-storage/index/dbox/dbox-map.h
index 81f7b5788904cb1e46a40c974a65dfbbb31731f6..751f69fa3320051a431465180ab6e1804440b310 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-map.h
+++ b/src/lib-storage/index/dbox/dbox-map.h
@@ -13,11 +13,13 @@ int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
 
 struct dbox_map_append_context *
 dbox_map_append_begin(struct dbox_mailbox *mbox);
-/* Request file for saving a new message with given size. If an existing file
-   can be used, the record is locked and updated in index. Returns 0 if ok,
-   -1 if error. */
+/* Request file for saving a new message with given size (if available). If an
+   existing file can be used, the record is locked and updated in index.
+   Returns 0 if ok, -1 if error. */
 int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size,
                         struct dbox_file **file_r, struct ostream **output_r);
+/* Finished saving the last mail. Saves the message size. */
+void dbox_map_append_finish_mail(struct dbox_map_append_context *ctx);
 /* Assign map UIDs to all appended msgs to multi-files. */
 int dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx,
                                    uint32_t *first_map_uid_r,

diff --git a/src/lib-storage/index/dbox/dbox-save.c b/src/lib-storage/index/dbox/dbox-save.c
index 964f5818bf051d30bef3b6638879d31481302d52..952f39b2468147373bf5748f621684fd6abc9289 100644 (file)
--- a/src/lib-storage/index/dbox/dbox-save.c
+++ b/src/lib-storage/index/dbox/dbox-save.c
@@ -272,6 +272,7 @@ static int dbox_save_finish_write(struct mail_save_context *_ctx)
                return -1;
        }
 
+       dbox_map_append_finish_mail(ctx->append_ctx);
        if (save_mail->file->single_mbox != NULL) {
                dbox_file_close(save_mail->file);
                ctx->single_count++;