Record size is allowed to change between index files. This will allow adding

extensions dynamically for existing indexes.

--HG--
branch : HEAD

diff --git a/src/lib-index/mail-index-fsck.c b/src/lib-index/mail-index-fsck.c
index b392065c38cb9db2b3d79416653d070733a082f4..75ec50f11462390907b44e68a31ebffec49ed3f5 100644 (file)
--- a/src/lib-index/mail-index-fsck.c
+++ b/src/lib-index/mail-index-fsck.c
@@ -75,7 +75,7 @@ static int mail_index_fsck_locked(struct mail_index *index,
                        hdr.first_deleted_uid_lowwater = rec->uid;
 
                last_uid = rec->uid;
-               rec = CONST_PTR_OFFSET(rec, index->record_size);
+               rec = CONST_PTR_OFFSET(rec, hdr.record_size);
        }
 
        if (hdr.next_uid <= last_uid) {
@@ -100,7 +100,7 @@ static int mail_index_fsck_locked(struct mail_index *index,
         CHECK(first_unseen_uid_lowwater, <);
         CHECK(first_deleted_uid_lowwater, <);
 
-       if (mail_index_write_header(index, &hdr) < 0)
+       if (mail_index_write_base_header(index, &hdr) < 0)
                return -1;
 
        return 1;

diff --git a/src/lib-index/mail-index-lock.c b/src/lib-index/mail-index-lock.c
index 4bc31a54ef2c142ac531f67b28e623e9680c4d52..88399394495de0437d8472927570ec212dd03ff4 100644 (file)
--- a/src/lib-index/mail-index-lock.c
+++ b/src/lib-index/mail-index-lock.c
@@ -187,7 +187,7 @@ static int mail_index_copy(struct mail_index *index)
        ret = write_full(fd, index->map->hdr, sizeof(*index->map->hdr));
        if (ret < 0 || write_full(fd, index->map->records,
                                  index->map->records_count *
-                                 index->record_size) < 0) {
+                                 index->map->hdr->record_size) < 0) {
                mail_index_file_set_syscall_error(index, path, "write_full()");
                (void)close(fd);
                (void)unlink(path);

diff --git a/src/lib-index/mail-index-private.h b/src/lib-index/mail-index-private.h
index ffd5a0b415165bc487f02938799190a6265043d1..def33dee504c12c52fea36e3c07b02802c7fc2c3 100644 (file)
--- a/src/lib-index/mail-index-private.h
+++ b/src/lib-index/mail-index-private.h
@@ -22,9 +22,15 @@ struct mail_transaction_header;
 #define MAIL_INDEX_MAP_IS_IN_MEMORY(map) \
        ((map)->buffer != NULL)
 
-#define MAIL_INDEX_MAP_IDX(index, map, idx) \
+#define MAIL_INDEX_MAP_IDX(map, idx) \
        ((struct mail_index_record *) \
-               PTR_OFFSET((map)->records, (idx) * (index)->record_size))
+               PTR_OFFSET((map)->records, (idx) * (map)->hdr->record_size))
+
+struct mail_index_extra_record_info {
+       const char *name;
+       uint16_t offset;
+       uint16_t size;
+};
 
 struct mail_index_map {
        int refcount;
@@ -33,6 +39,9 @@ struct mail_index_map {
        void *records; /* struct mail_index_record[] */
        unsigned int records_count;
 
+       struct mail_index_extra_record_info *
+               extra_record_map[MAIL_INDEX_MAX_EXTRA_RECORDS];
+
        void *mmap_base;
        size_t mmap_size, mmap_used_size;
 
@@ -46,12 +55,6 @@ struct mail_index_map {
        unsigned int write_to_disk:1;
 };
 
-struct mail_index_extra_record_info {
-       const char *name;
-       uint16_t offset;
-       uint16_t size;
-};
-
 struct mail_index {
        char *dir, *prefix;
 
@@ -65,8 +68,7 @@ struct mail_index {
        buffer_t *extra_records_buf;
        const struct mail_index_extra_record_info *extra_records;
        unsigned int extra_records_count;
-
-       unsigned int record_size;
+       unsigned int max_record_size;
 
        char *filepath;
        int fd;
@@ -95,8 +97,8 @@ struct mail_index {
        unsigned int fsck:1;
 };
 
-int mail_index_write_header(struct mail_index *index,
-                           const struct mail_index_header *hdr);
+int mail_index_write_base_header(struct mail_index *index,
+                                const struct mail_index_header *hdr);
 
 int mail_index_reopen(struct mail_index *index, int fd);
 int mail_index_create_tmp_file(struct mail_index *index, const char **path_r);
@@ -122,7 +124,7 @@ int mail_index_map(struct mail_index *index, int force);
 /* Unreference given mapping and unmap it if it's dropped to zero. */
 void mail_index_unmap(struct mail_index *index, struct mail_index_map *map);
 struct mail_index_map *
-mail_index_map_to_memory(struct mail_index *index, struct mail_index_map *map);
+mail_index_map_to_memory(struct mail_index_map *map, uint32_t new_record_size);
 
 int mail_index_lookup_full(struct mail_index_view *view, uint32_t seq,
                           struct mail_index_map **map_r,

diff --git a/src/lib-index/mail-index-sync-private.h b/src/lib-index/mail-index-sync-private.h
index f7e02e9bc4d66466d1f82939afa2148482490138..e3260c4a1f3951a7ed78ca1d3615869b8228021e 100644 (file)
--- a/src/lib-index/mail-index-sync-private.h
+++ b/src/lib-index/mail-index-sync-private.h
@@ -5,12 +5,14 @@ struct mail_index_sync_ctx {
        struct mail_index *index;
        struct mail_index_view *view;
 
-       buffer_t *expunges_buf, *updates_buf, *appends_buf;
+       buffer_t *expunges_buf, *updates_buf;
 
        const struct mail_transaction_expunge *expunges;
        const struct mail_transaction_flag_update *updates;
        size_t expunges_count, updates_count;
 
+       uint32_t append_uid_first, append_uid_last;
+
        const struct mail_transaction_header *hdr;
        const void *data;
 

diff --git a/src/lib-index/mail-index-sync-update.c b/src/lib-index/mail-index-sync-update.c
index 5c20955b3263b50c2d5171f3bef6f0e7e0f80a5d..ce963951cd303fc31459090a29d2023324ff3a28 100644 (file)
--- a/src/lib-index/mail-index-sync-update.c
+++ b/src/lib-index/mail-index-sync-update.c
@@ -56,7 +56,6 @@ mail_index_header_update_lowwaters(struct mail_index_header *hdr,
 static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
 {
        struct mail_index_view *view = context;
-       struct mail_index *index = view->index;
        struct mail_index_map *map = view->map;
        struct mail_index_header *hdr = &map->hdr_copy;
        struct mail_index_record *rec;
@@ -72,15 +71,14 @@ static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
                return 1;
 
        for (seq = seq1; seq <= seq2; seq++) {
-                rec = MAIL_INDEX_MAP_IDX(index, map, seq-1);
+                rec = MAIL_INDEX_MAP_IDX(map, seq-1);
                mail_index_header_update_counts(hdr, rec->flags, 0);
        }
 
        /* @UNSAFE */
        count = seq2 - seq1 + 1;
-       memmove(MAIL_INDEX_MAP_IDX(index, map, seq1-1),
-               MAIL_INDEX_MAP_IDX(index, map, seq2),
-               (map->records_count - seq2) * view->index->record_size);
+       memmove(MAIL_INDEX_MAP_IDX(map, seq1-1), MAIL_INDEX_MAP_IDX(map, seq2),
+               (map->records_count - seq2) * map->hdr->record_size);
 
        map->records_count -= count;
        hdr->messages_count -= count;
@@ -88,17 +86,20 @@ static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
 
        if (map->buffer != NULL) {
                buffer_set_used_size(map->buffer, map->records_count *
-                                    view->index->record_size);
+                                    map->hdr->record_size);
                map->records = buffer_get_modifyable_data(map->buffer, NULL);
        }
        return 1;
 }
 
-static int sync_append(const struct mail_index_record *rec, void *context)
+static int sync_append(const struct mail_transaction_append_header *hdr,
+                      const struct mail_index_record *rec, void *context)
 {
        struct mail_index_view *view = context;
-       struct mail_index *index = view->index;
        struct mail_index_map *map = view->map;
+       void *dest;
+
+       i_assert(hdr->record_size <= map->hdr->record_size);
 
        if (rec->uid < map->hdr_copy.next_uid) {
                mail_transaction_log_view_set_corrupted(view->log_view,
@@ -108,16 +109,19 @@ static int sync_append(const struct mail_index_record *rec, void *context)
        }
 
        if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
-               i_assert(map->records_count * index->record_size ==
+               i_assert(map->records_count * map->hdr->record_size ==
                         buffer_get_used_size(map->buffer));
-               buffer_append(map->buffer, rec, index->record_size);
+               dest = buffer_append_space_unsafe(map->buffer,
+                                                 map->hdr->record_size);
                map->records = buffer_get_modifyable_data(map->buffer, NULL);
        } else {
-               i_assert((map->records_count+1) * index->record_size <=
+               i_assert((map->records_count+1) * map->hdr->record_size <=
                         map->mmap_size);
-               memcpy(MAIL_INDEX_MAP_IDX(index, map, map->records_count),
-                      rec, index->record_size);
+               dest = MAIL_INDEX_MAP_IDX(map, map->records_count);
        }
+       memcpy(dest, rec, hdr->record_size);
+       memset(PTR_OFFSET(dest, hdr->record_size), 0,
+              map->hdr->record_size - hdr->record_size);
 
        map->hdr_copy.messages_count++;
        map->hdr_copy.next_uid = rec->uid+1;
@@ -161,7 +165,7 @@ static int sync_flag_update(const struct mail_transaction_flag_update *u,
         flag_mask = ~u->remove_flags;
 
        for (idx = seq1-1; idx < seq2; idx++) {
-                rec = MAIL_INDEX_MAP_IDX(view->index, view->map, idx);
+                rec = MAIL_INDEX_MAP_IDX(view->map, idx);
 
                old_flags = rec->flags;
                rec->flags = (rec->flags & flag_mask) | u->add_flags;
@@ -187,7 +191,7 @@ static int sync_cache_reset(const struct mail_transaction_cache_reset *u,
        view->map->hdr_copy.cache_file_seq = u->new_file_seq;
 
        for (i = 0; i < view->messages_count; i++)
-               MAIL_INDEX_MAP_IDX(view->index, view->map, i)->cache_offset = 0;
+               MAIL_INDEX_MAP_IDX(view->map, i)->cache_offset = 0;
        return 1;
 }
 
@@ -203,8 +207,8 @@ static int sync_cache_update(const struct mail_transaction_cache_update *u,
        i_assert(ret == 0);
 
        if (seq != 0) {
-               MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1)->
-                       cache_offset = u->cache_offset;
+               MAIL_INDEX_MAP_IDX(view->map, seq-1)->cache_offset =
+                       u->cache_offset;
        }
        return 1;
 }
@@ -241,7 +245,7 @@ sync_extra_rec_update(const struct mail_transaction_extra_rec_header *hdr,
                offset = view->index->extra_records[hdr->idx].offset;
                size = view->index->extra_records[hdr->idx].size;
 
-               rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1);
+               rec = MAIL_INDEX_MAP_IDX(view->map, seq-1);
                memcpy(PTR_OFFSET(rec, offset), u->data, size);
        }
        return 1;
@@ -259,7 +263,7 @@ static int mail_index_grow(struct mail_index *index, struct mail_index_map *map,
        i_assert(map == index->map);
 
        size = map->hdr->header_size +
-               (map->records_count + count) * index->record_size;
+               (map->records_count + count) * map->hdr->record_size;
        if (size <= map->mmap_size)
                return 0;
 
@@ -271,7 +275,7 @@ static int mail_index_grow(struct mail_index *index, struct mail_index_map *map,
        index->last_grow_count = count;
 
        size = map->hdr->header_size +
-               (map->records_count + count) * index->record_size;
+               (map->records_count + count) * map->hdr->record_size;
        if (file_set_size(index->fd, (off_t)size) < 0)
                return mail_index_set_syscall_error(index, "file_set_size()");
 
@@ -290,6 +294,18 @@ static int mail_index_grow(struct mail_index *index, struct mail_index_map *map,
        return 0;
 }
 
+static void mail_index_sync_replace_map(struct mail_index_view *view,
+                                       struct mail_index_map *map)
+{
+       mail_index_unmap(view->index, view->map);
+       view->map = map;
+       view->map->refcount++;
+       mail_index_unmap(view->index, view->index->map);
+       view->index->map = map;
+       view->index->hdr = map->hdr;
+       map->write_to_disk = TRUE;
+}
+
 int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
 {
        struct mail_index *index = sync_ctx->index;
@@ -335,18 +351,23 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
                           replace the whole index file. to avoid extra disk
                           I/O we copy the index into memory rather than to
                           temporary file */
-                       map = mail_index_map_to_memory(index, map);
-                       mail_index_unmap(index, view->map);
-                       view->map = map;
-                       view->map->refcount++;
-                       mail_index_unmap(index, index->map);
-                       index->map = map;
-                       index->hdr = map->hdr;
-                       map->write_to_disk = TRUE;
+                       map = mail_index_map_to_memory(map,
+                                                      map->hdr->record_size);
+                       mail_index_sync_replace_map(view, map);
                }
 
                if ((hdr->type & MAIL_TRANSACTION_APPEND) != 0) {
-                       count = hdr->size / index->record_size;
+                        const struct mail_transaction_append_header *append_hdr;
+
+                       append_hdr = data;
+                       if (append_hdr->record_size > map->hdr->record_size) {
+                               /* we have to grow our record size */
+                               map = mail_index_map_to_memory(map,
+                                       append_hdr->record_size);
+                               mail_index_sync_replace_map(view, map);
+                       }
+                       count = (hdr->size - sizeof(*append_hdr)) /
+                                append_hdr->record_size;
                        if (mail_index_grow(index, view->map, count) < 0) {
                                ret = -1;
                                break;
@@ -381,7 +402,7 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
                const struct mail_index_record *rec;
 
                for (i = 0; i < map->records_count; i++) {
-                       rec = MAIL_INDEX_MAP_IDX(index, map, i);
+                       rec = MAIL_INDEX_MAP_IDX(map, i);
                        if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
                                map->hdr_copy.flags |=
                                        MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
@@ -392,7 +413,7 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
 
        if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
                map->mmap_used_size = index->hdr->header_size +
-                       map->records_count * index->record_size;
+                       map->records_count * map->hdr->record_size;
 
                memcpy(map->mmap_base, &map->hdr_copy, sizeof(map->hdr_copy));
                if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) {

diff --git a/src/lib-index/mail-index-sync.c b/src/lib-index/mail-index-sync.c
index 1bc89ecfbc7626b63c3c523c10b56ae93a578439..885dd0e92eff1e47e156bcc5a0ff21d8d0f17c97 100644 (file)
--- a/src/lib-index/mail-index-sync.c
+++ b/src/lib-index/mail-index-sync.c
@@ -80,11 +80,23 @@ static void mail_index_sync_sort_transaction(struct mail_index_sync_ctx *ctx)
                        mail_index_sync_sort_flags(ctx);
                }
                break;
-       case MAIL_TRANSACTION_APPEND:
-               buffer_append(ctx->appends_buf, ctx->data, ctx->hdr->size);
+       case MAIL_TRANSACTION_APPEND: {
+               const struct mail_transaction_append_header *hdr = ctx->data;
+               const struct mail_index_record *rec = ctx->data;
+
+               if (ctx->append_uid_first == 0 ||
+                   rec->uid < ctx->append_uid_first)
+                       ctx->append_uid_first = rec->uid;
+
+               rec = CONST_PTR_OFFSET(ctx->data,
+                                      ctx->hdr->size - hdr->record_size);
+               if (rec->uid > ctx->append_uid_last)
+                       ctx->append_uid_last = rec->uid;
+
                 ctx->sync_appends = TRUE;
                break;
        }
+       }
 }
 
 static int mail_index_sync_read_and_sort(struct mail_index_sync_ctx *ctx)
@@ -172,8 +184,6 @@ int mail_index_sync_begin(struct mail_index *index,
                                                  1024, (size_t)-1);
        ctx->updates_buf = buffer_create_dynamic(default_pool,
                                                 1024, (size_t)-1);
-       ctx->appends_buf = buffer_create_dynamic(default_pool,
-                                                1024, (size_t)-1);
        if (mail_index_sync_read_and_sort(ctx) < 0) {
                 mail_index_sync_end(ctx);
                return -1;
@@ -304,12 +314,8 @@ int mail_index_sync_next(struct mail_index_sync_ctx *ctx,
        if (ctx->sync_appends) {
                ctx->sync_appends = FALSE;
                sync_rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
-               sync_rec->appends = buffer_get_data(ctx->appends_buf,
-                                                   &sync_rec->appends_count);
-               sync_rec->appends_count /= ctx->index->record_size;
-               sync_rec->uid1 = sync_rec->appends[0].uid;
-               sync_rec->uid2 =
-                       sync_rec->appends[sync_rec->appends_count-1].uid;
+               sync_rec->uid1 = ctx->append_uid_first;
+               sync_rec->uid2 = ctx->append_uid_last;
                return 1;
        }
 
@@ -355,8 +361,6 @@ int mail_index_sync_end(struct mail_index_sync_ctx *ctx)
                buffer_free(ctx->expunges_buf);
        if (ctx->updates_buf != NULL)
                buffer_free(ctx->updates_buf);
-       if (ctx->appends_buf != NULL)
-               buffer_free(ctx->appends_buf);
        i_free(ctx);
        return ret;
 }

diff --git a/src/lib-index/mail-index-transaction-private.h b/src/lib-index/mail-index-transaction-private.h
index f22ac491ca375e99f2c487f32799e9f9dd41b4be..64808fd1da8a45562601d9c3740fac9ed63f3678 100644 (file)
--- a/src/lib-index/mail-index-transaction-private.h
+++ b/src/lib-index/mail-index-transaction-private.h
@@ -8,6 +8,7 @@ struct mail_index_transaction {
 
         buffer_t *appends;
        uint32_t first_new_seq, last_new_seq;
+       unsigned int append_record_size;
 
        buffer_t *expunges;
 

diff --git a/src/lib-index/mail-index-transaction-view.c b/src/lib-index/mail-index-transaction-view.c
index 30c1bb022eb38eedfb82336c98fbce3085a60123..e4a56fce16d253776f696bf216d240a235e995d5 100644 (file)
--- a/src/lib-index/mail-index-transaction-view.c
+++ b/src/lib-index/mail-index-transaction-view.c
@@ -130,7 +130,7 @@ static int _tview_lookup_first(struct mail_index_view *view,
                        *seq_r = seq;
                        break;
                }
-               rec = CONST_PTR_OFFSET(rec, view->index->record_size);
+               rec = CONST_PTR_OFFSET(rec, view->index->max_record_size);
        }
 
        return 0;

diff --git a/src/lib-index/mail-index-transaction.c b/src/lib-index/mail-index-transaction.c
index 58e7766388a70bd605fc20d8ad304cae8312deeb..0084baaae9d57bf7c3dc4faa2213ae467f44de5e 100644 (file)
--- a/src/lib-index/mail-index-transaction.c
+++ b/src/lib-index/mail-index-transaction.c
@@ -70,12 +70,10 @@ mail_index_buffer_convert_to_uids(struct mail_index_view *view,
                seq = (uint32_t *)&data[i];
                i_assert(seq[0] <= view->map->records_count);
 
-               seq[0] = MAIL_INDEX_MAP_IDX(view->index, view->map,
-                                           seq[0]-1)->uid;
+               seq[0] = MAIL_INDEX_MAP_IDX(view->map, seq[0]-1)->uid;
                if (range) {
                        i_assert(seq[1] <= view->map->records_count);
-                       seq[1] = MAIL_INDEX_MAP_IDX(view->index, view->map,
-                                                   seq[1]-1)->uid;
+                       seq[1] = MAIL_INDEX_MAP_IDX(view->map, seq[1]-1)->uid;
                }
        }
 }
@@ -139,6 +137,40 @@ void mail_index_transaction_rollback(struct mail_index_transaction *t)
         mail_index_transaction_free(t);
 }
 
+static void
+mail_index_transaction_update_append_size(struct mail_index_transaction *t)
+{
+       buffer_t *new_buf;
+       unsigned int new_record_size;
+       const void *src;
+       void *dest;
+       size_t i, size;
+
+       new_record_size = t->view->index->max_record_size;
+       if (t->append_record_size == new_record_size)
+               return;
+
+       i_assert(t->append_record_size < new_record_size);
+
+       if (t->append_record_size != 0) {
+               /* resize the records in buffer */
+               src = buffer_get_data(t->appends, &size);
+               size /= t->append_record_size;
+
+               new_buf = buffer_create_dynamic(default_pool,
+                                               size * new_record_size,
+                                               (size_t)-1);
+               for (i = 0; i < size; i++) {
+                       dest = buffer_append_space_unsafe(new_buf,
+                                                         new_record_size);
+                       memcpy(dest, src, t->append_record_size);
+                       src = CONST_PTR_OFFSET(src, t->append_record_size);
+               }
+       }
+
+       t->append_record_size = new_record_size;
+}
+
 struct mail_index_record *
 mail_index_transaction_lookup(struct mail_index_transaction *t, uint32_t seq)
 {
@@ -146,9 +178,10 @@ mail_index_transaction_lookup(struct mail_index_transaction *t, uint32_t seq)
 
        i_assert(seq >= t->first_new_seq && seq <= t->last_new_seq);
 
-       pos = (seq - t->first_new_seq) * t->view->index->record_size;
-       return buffer_get_space_unsafe(t->appends, pos,
-                                      t->view->index->record_size);
+       mail_index_transaction_update_append_size(t);
+
+       pos = (seq - t->first_new_seq) * t->append_record_size;
+       return buffer_get_space_unsafe(t->appends, pos, t->append_record_size);
 }
 
 void mail_index_append(struct mail_index_transaction *t, uint32_t uid,
@@ -159,7 +192,9 @@ void mail_index_append(struct mail_index_transaction *t, uint32_t uid,
        if (t->appends == NULL) {
                t->appends = buffer_create_dynamic(default_pool,
                                                   4096, (size_t)-1);
+               t->append_record_size = t->view->index->max_record_size;
        }
+       mail_index_transaction_update_append_size(t);
 
        /* sequence number is visible only inside given view,
           so let it generate it */
@@ -168,9 +203,8 @@ void mail_index_append(struct mail_index_transaction *t, uint32_t uid,
        else
                *seq_r = t->last_new_seq = t->first_new_seq;
 
-       rec = buffer_append_space_unsafe(t->appends,
-                                        t->view->index->record_size);
-       memset(rec, 0, t->view->index->record_size);
+       rec = buffer_append_space_unsafe(t->appends, t->append_record_size);
+       memset(rec, 0, t->append_record_size);
        rec->uid = uid;
 }
 

diff --git a/src/lib-index/mail-index-view-sync.c b/src/lib-index/mail-index-view-sync.c
index 7c47601c7910aff789375a2a7bc1eead7ce9af07..cc9a764d8dc86e72b4ec5f3fafb3a4b336cd19f5 100644 (file)
--- a/src/lib-index/mail-index-view-sync.c
+++ b/src/lib-index/mail-index-view-sync.c
@@ -125,7 +125,8 @@ int mail_index_view_sync_begin(struct mail_index_view *view,
                if (view->map != view->index->map)
                        ctx->sync_map_update = TRUE;
 
-               map = mail_index_map_to_memory(view->index, view->map);
+               map = mail_index_map_to_memory(view->map,
+                                              view->map->hdr->record_size);
                mail_index_unmap(view->index, view->map);
                view->map = map;
        }

diff --git a/src/lib-index/mail-index-view.c b/src/lib-index/mail-index-view.c
index 479249a13e800b488882a36d0106a5f11cf1dabc..f5fa8946170deb4f17a4f993863a46cf6fd1f7dc 100644 (file)
--- a/src/lib-index/mail-index-view.c
+++ b/src/lib-index/mail-index-view.c
@@ -178,7 +178,7 @@ static int _view_lookup_full(struct mail_index_view *view, uint32_t seq,
        if (mail_index_view_lock(view) < 0)
                return -1;
 
-       rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1);
+       rec = MAIL_INDEX_MAP_IDX(view->map, seq-1);
        if (view->map == view->index->map) {
                *map_r = view->map;
                *rec_r = rec;
@@ -203,7 +203,7 @@ static int _view_lookup_full(struct mail_index_view *view, uint32_t seq,
        do {
                // FIXME: we could be skipping more by uid diff
                seq--;
-               n_rec = MAIL_INDEX_MAP_IDX(view->index, map, seq);
+               n_rec = MAIL_INDEX_MAP_IDX(map, seq);
                if (n_rec->uid <= uid)
                        break;
        } while (seq > 0);
@@ -225,7 +225,7 @@ static int _view_lookup_uid(struct mail_index_view *view, uint32_t seq,
        if (mail_index_view_lock(view) < 0)
                return -1;
 
-       *uid_r = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1)->uid;
+       *uid_r = MAIL_INDEX_MAP_IDX(view->map, seq-1)->uid;
        return 0;
 }
 
@@ -239,7 +239,7 @@ static uint32_t mail_index_bsearch_uid(struct mail_index_view *view,
        i_assert(view->messages_count <= view->map->records_count);
 
        rec_base = view->map->records;
-       record_size = view->index->record_size;
+       record_size = view->map->hdr->record_size;
 
        idx = left_idx = *left_idx_p;
        right_idx = view->messages_count;
@@ -301,8 +301,7 @@ static int _view_lookup_uid_range(struct mail_index_view *view,
        left_idx = 0;
        *first_seq_r = mail_index_bsearch_uid(view, first_uid, &left_idx, 1);
        if (*first_seq_r == 0 ||
-           MAIL_INDEX_MAP_IDX(view->index, view->map, *first_seq_r-1)->uid >
-           last_uid) {
+           MAIL_INDEX_MAP_IDX(view->map, *first_seq_r-1)->uid > last_uid) {
                *first_seq_r = 0;
                *last_seq_r = 0;
                return 0;
@@ -351,7 +350,7 @@ static int _view_lookup_first(struct mail_index_view *view,
        }
 
        for (; seq <= view->messages_count; seq++) {
-               rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1);
+               rec = MAIL_INDEX_MAP_IDX(view->map, seq-1);
                if ((rec->flags & flags_mask) == (uint8_t)flags) {
                        *seq_r = seq;
                        break;

diff --git a/src/lib-index/mail-index.c b/src/lib-index/mail-index.c
index 3093020df70ec0ffc9e1552735494d14a4227dc1..a57f69a88040b0a64f08d4a455d342245fb4345d 100644 (file)
--- a/src/lib-index/mail-index.c
+++ b/src/lib-index/mail-index.c
@@ -31,7 +31,7 @@ struct mail_index *mail_index_alloc(const char *dir, const char *prefix)
        index->extra_records_buf =
                buffer_create_dynamic(index->extra_records_pool,
                                      64, (size_t)-1);
-       index->record_size = sizeof(struct mail_index_record);
+       index->max_record_size = sizeof(struct mail_index_record);
 
        index->mode = 0600;
        index->gid = (gid_t)-1;
@@ -66,7 +66,7 @@ uint32_t mail_index_register_record_extra(struct mail_index *index,
 
        i_assert(size % 4 == 0);
        i_assert(!index->opened);
-       i_assert(index->record_size + size <= 65535);
+       i_assert(index->max_record_size + size <= 65535);
 
        if (index->extra_records_count >= MAIL_INDEX_MAX_EXTRA_RECORDS) {
                i_panic("Maximum extra record count reached, "
@@ -78,14 +78,14 @@ uint32_t mail_index_register_record_extra(struct mail_index *index,
        memset(&info, 0, sizeof(info));
        info.name = p_strdup(index->extra_records_pool, name);
        info.size = size;
-       info.offset = index->record_size;
+       info.offset = index->max_record_size;
 
        buffer_append(index->extra_records_buf, &info, sizeof(info));
        index->extra_records =
                buffer_get_data(index->extra_records_buf, &buf_size);
        index->extra_records_count = buf_size / sizeof(info);
 
-       index->record_size += size;
+       index->max_record_size += size;
        return index->extra_records_count-1;
 }
 
@@ -132,11 +132,11 @@ static int mail_index_check_header(struct mail_index *index,
                return -1;
        }
 
-       if (hdr->record_size != index->record_size) {
+       if (hdr->record_size < sizeof(struct mail_index_record)) {
                mail_index_set_error(index, "Corrupted index file %s: "
-                                    "record_size mismatch: %d != %d",
+                                    "record_size too small: %u < %"PRIuSIZE_T,
                                     index->filepath, hdr->record_size,
-                                    (int)index->record_size);
+                                    sizeof(struct mail_index_record));
                return -1;
        }
 
@@ -227,11 +227,11 @@ static int mail_index_mmap(struct mail_index *index, struct mail_index_map *map)
        hdr = map->mmap_base;
        map->hdr = hdr;
        map->mmap_used_size = hdr->header_size +
-               hdr->messages_count * index->record_size;
+               hdr->messages_count * hdr->record_size;
 
        if (map->mmap_used_size > map->mmap_size) {
                records_count = (map->mmap_size - hdr->header_size) /
-                       index->record_size;
+                       hdr->record_size;
                mail_index_set_error(index, "Corrupted index file %s: "
                                     "messages_count too large (%u > %u)",
                                     index->filepath, hdr->messages_count,
@@ -271,7 +271,7 @@ static int mail_index_read_map(struct mail_index *index,
                        pos += ret;
        }
        if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE) {
-               records_size = hdr.messages_count * index->record_size;
+               records_size = hdr.messages_count * hdr.record_size;
 
                if (map->buffer == NULL) {
                        map->buffer = buffer_create_dynamic(default_pool,
@@ -367,7 +367,7 @@ int mail_index_map(struct mail_index *index, int force)
                        return -1;
 
                used_size = hdr->header_size +
-                       hdr->messages_count * index->record_size;
+                       hdr->messages_count * hdr->record_size;
                if (map->mmap_size >= used_size && !force) {
                        map->records_count = hdr->messages_count;
                        return 1;
@@ -410,27 +410,41 @@ int mail_index_map(struct mail_index *index, int force)
 }
 
 struct mail_index_map *
-mail_index_map_to_memory(struct mail_index *index, struct mail_index_map *map)
+mail_index_map_to_memory(struct mail_index_map *map, uint32_t new_record_size)
 {
        struct mail_index_map *mem_map;
-       size_t size;
+       void *src, *dest;
+       size_t size, copy_size;
+       unsigned int i;
 
        if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
                map->refcount++;
                return map;
        }
 
-        size = map->records_count * index->record_size;
+        size = map->records_count * new_record_size;
 
        mem_map = i_new(struct mail_index_map, 1);
        mem_map->refcount = 1;
        mem_map->buffer = buffer_create_dynamic(default_pool, size, (size_t)-1);
-       buffer_append(mem_map->buffer, map->records, size);
+       if (map->hdr->record_size == new_record_size)
+               buffer_append(mem_map->buffer, map->records, size);
+       else {
+               copy_size = I_MIN(map->hdr->record_size, new_record_size);
+               src = map->records;
+               for (i = 0; i < map->records_count; i++) {
+                       dest = buffer_append_space_unsafe(mem_map->buffer,
+                                                         new_record_size);
+                       memcpy(dest, src, copy_size);
+                       src = PTR_OFFSET(src, map->hdr->record_size);
+               }
+       }
 
        mem_map->records = buffer_get_modifyable_data(mem_map->buffer, NULL);
        mem_map->records_count = map->records_count;
 
        mem_map->hdr_copy = *map->hdr;
+       mem_map->hdr_copy.record_size = new_record_size;
        mem_map->hdr = &mem_map->hdr_copy;
        return mem_map;
 }
@@ -497,15 +511,19 @@ mail_index_try_open(struct mail_index *index, unsigned int *lock_id_r)
        return ret;
 }
 
-int mail_index_write_header(struct mail_index *index,
-                           const struct mail_index_header *hdr)
+int mail_index_write_base_header(struct mail_index *index,
+                                const struct mail_index_header *hdr)
 {
+       size_t hdr_size;
+
+       hdr_size = I_MIN(sizeof(*hdr), hdr->base_header_size);
+
        if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
-               memcpy(index->map->mmap_base, hdr, sizeof(*hdr));
-               if (msync(index->map->mmap_base, sizeof(*hdr), MS_SYNC) < 0)
+               memcpy(index->map->mmap_base, hdr, hdr_size);
+               if (msync(index->map->mmap_base, hdr_size, MS_SYNC) < 0)
                        return mail_index_set_syscall_error(index, "msync()");
        } else {
-               if (pwrite_full(index->fd, hdr, sizeof(*hdr), 0) < 0) {
+               if (pwrite_full(index->fd, hdr, hdr_size, 0) < 0) {
                        mail_index_set_syscall_error(index, "pwrite_full()");
                        return -1;
                }
@@ -602,7 +620,7 @@ static void mail_index_header_init(struct mail_index *index,
        hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
        hdr->base_header_size = sizeof(*hdr);
        hdr->header_size = sizeof(*hdr);
-       hdr->record_size = index->record_size;
+       hdr->record_size = index->max_record_size;
        hdr->keywords_mask_size = sizeof(keywords_mask_t);
 
 #ifndef WORDS_BIGENDIAN
@@ -890,7 +908,7 @@ void mail_index_mark_corrupted(struct mail_index *index)
 
        hdr = *index->hdr;
        hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED;
-       if (mail_index_write_header(index, &hdr) == 0) {
+       if (mail_index_write_base_header(index, &hdr) == 0) {
                if (fsync(index->fd) < 0)
                        mail_index_set_syscall_error(index, "fsync()");
        }

diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h
index 365de0fc63cf5cc576f74a26819a983a9a5b38be..9835508aa6a0ee12e61b6029eb91b5d1d6210ce2 100644 (file)
--- a/src/lib-index/mail-index.h
+++ b/src/lib-index/mail-index.h
@@ -123,10 +123,6 @@ struct mail_index_sync_rec {
        keywords_mask_t add_keywords;
        uint8_t remove_flags;
        keywords_mask_t remove_keywords;
-
-       /* MAIL_INDEX_SYNC_TYPE_APPEND: */
-       const struct mail_index_record *appends;
-       size_t appends_count;
 };
 
 struct mail_index;

diff --git a/src/lib-index/mail-transaction-log.c b/src/lib-index/mail-transaction-log.c
index bcf4f7e1c984e91aeb69492f5865f61bc1a05ef8..b9637813203a3dd0b798ffe1ceb784e5e6b2045b 100644 (file)
--- a/src/lib-index/mail-transaction-log.c
+++ b/src/lib-index/mail-transaction-log.c
@@ -874,6 +874,7 @@ static int mail_transaction_log_fix_appends(struct mail_transaction_log *log,
        struct mail_transaction_log_view *sync_view;
        const struct mail_index_record *old, *old_end;
        struct mail_index_record *appends, *end, *rec, *dest;
+        const struct mail_transaction_append_header *append_hdr;
        const struct mail_transaction_header *hdr;
        const void *data;
        size_t size;
@@ -883,7 +884,7 @@ static int mail_transaction_log_fix_appends(struct mail_transaction_log *log,
        if (t->appends == NULL)
                return 0;
 
-       record_size = log->index->record_size;
+       record_size = t->append_record_size;
        appends = buffer_get_modifyable_data(t->appends, &size);
        end = PTR_OFFSET(appends, size);
 
@@ -905,8 +906,10 @@ static int mail_transaction_log_fix_appends(struct mail_transaction_log *log,
                    MAIL_TRANSACTION_APPEND)
                        continue;
 
-               old = data;
-               old_end = CONST_PTR_OFFSET(old, hdr->size);
+                append_hdr = data;
+
+               old = CONST_PTR_OFFSET(data, sizeof(*append_hdr));
+               old_end = CONST_PTR_OFFSET(data, hdr->size);
                while (old != old_end) {
                        /* appends are sorted */
                        for (rec = appends; rec != end; ) {
@@ -919,7 +922,7 @@ static int mail_transaction_log_fix_appends(struct mail_transaction_log *log,
                                }
                                rec = PTR_OFFSET(rec, record_size);
                        }
-                        old = CONST_PTR_OFFSET(old, record_size);
+                        old = CONST_PTR_OFFSET(old, append_hdr->record_size);
                }
        }
 
@@ -1097,7 +1100,15 @@ int mail_transaction_log_append(struct mail_index_transaction *t,
 
        ret = 0;
        if (t->appends != NULL) {
-               ret = log_append_buffer(file, t->appends, NULL,
+               struct mail_transaction_append_header hdr;
+
+               memset(&hdr, 0, sizeof(hdr));
+               hdr.record_size = t->append_record_size;
+
+               hdr_buf = buffer_create_data(pool_datastack_create(),
+                                            &hdr, sizeof(hdr));
+               buffer_set_used_size(hdr_buf, sizeof(hdr));
+               ret = log_append_buffer(file, t->appends, hdr_buf,
                                        MAIL_TRANSACTION_APPEND,
                                        view->external);
        }

diff --git a/src/lib-index/mail-transaction-log.h b/src/lib-index/mail-transaction-log.h
index 572f7a7ee5977a8a7eb255e7111f3da9950ebc24..ce2198e582b2d7fda46ad3cdfbb715e17a449c34 100644 (file)
--- a/src/lib-index/mail-transaction-log.h
+++ b/src/lib-index/mail-transaction-log.h
@@ -75,6 +75,10 @@ struct mail_transaction_extra_rec_update {
        unsigned char data[1]; /* variable size */
 };
 
+struct mail_transaction_append_header {
+       uint32_t record_size;
+};
+
 struct mail_transaction_log *
 mail_transaction_log_open_or_create(struct mail_index *index);
 void mail_transaction_log_close(struct mail_transaction_log *log);

diff --git a/src/lib-index/mail-transaction-util.c b/src/lib-index/mail-transaction-util.c
index cb102b29c23b4b107957db55788662ee5cd05892..5fa3e7683e4c17f4c2c55b0f873f26d5dbcae6cc 100644 (file)
--- a/src/lib-index/mail-transaction-util.c
+++ b/src/lib-index/mail-transaction-util.c
@@ -60,23 +60,23 @@ int mail_transaction_map(struct mail_index *index,
                         struct mail_transaction_map_functions *map,
                         void *context)
 {
-
        int ret = 0;
 
        switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
        case MAIL_TRANSACTION_APPEND: {
+                const struct mail_transaction_append_header *append_hdr = data;
                const struct mail_index_record *rec, *end;
-               uint32_t record_size = index->record_size;
 
                if (map->append == NULL)
                        break;
 
+               rec = CONST_PTR_OFFSET(data, sizeof(*append_hdr));
                end = CONST_PTR_OFFSET(data, hdr->size);
-               for (rec = data; rec != end; ) {
-                       ret = map->append(rec, context);
+               while (rec != end) {
+                       ret = map->append(append_hdr, rec, context);
                        if (ret <= 0)
                                break;
-                       rec = CONST_PTR_OFFSET(rec, record_size);
+                       rec = CONST_PTR_OFFSET(rec, append_hdr->record_size);
                }
                break;
        }

diff --git a/src/lib-index/mail-transaction-util.h b/src/lib-index/mail-transaction-util.h
index 45303fbd1dfcf5236fb79c37515e7204750c7de9..420518d53464fbe8df76153d9359447c707abb51 100644 (file)
--- a/src/lib-index/mail-transaction-util.h
+++ b/src/lib-index/mail-transaction-util.h
@@ -10,7 +10,8 @@ extern const struct mail_transaction_type_map mail_transaction_type_map[];
 
 struct mail_transaction_map_functions {
        int (*expunge)(const struct mail_transaction_expunge *e, void *context);
-       int (*append)(const struct mail_index_record *rec, void *context);
+       int (*append)(const struct mail_transaction_append_header *hdr,
+                     const struct mail_index_record *rec, void *context);
        int (*flag_update)(const struct mail_transaction_flag_update *u,
                           void *context);
        int (*cache_reset)(const struct mail_transaction_cache_reset *u,