]> git.ipfire.org Git - thirdparty/dovecot/core.git/commitdiff
Caching fixes and optimizations. Removed all network byte ordering code -
authorTimo Sirainen <tss@iki.fi>
Sun, 4 Jul 2004 14:26:22 +0000 (17:26 +0300)
committerTimo Sirainen <tss@iki.fi>
Sun, 4 Jul 2004 14:26:22 +0000 (17:26 +0300)
it's not worth the trouble and would require massive changes in indexing
code as well to be useful. Changed next_offset to prev_offset which is
updated while syncing index.

--HG--
branch : HEAD

src/lib-index/mail-cache-compress.c
src/lib-index/mail-cache-lookup.c
src/lib-index/mail-cache-private.h
src/lib-index/mail-cache-transaction.c
src/lib-index/mail-cache.c
src/lib-index/mail-cache.h
src/lib-index/mail-index-sync-private.h
src/lib-index/mail-index-sync-update.c
src/lib-storage/index/index-mail-headers.c
src/lib-storage/index/index-transaction.c

index 555ff7ffd8f055b3fbc58155e2dcbb31dad785c6..5736ae3ee240fc6356600509112f787fa20a37d5 100644 (file)
@@ -2,7 +2,6 @@
 
 #include "lib.h"
 #include "buffer.h"
-#include "byteorder.h"
 #include "ostream.h"
 #include "file-set-size.h"
 #include "mail-cache-private.h"
@@ -19,7 +18,7 @@ mail_cache_compress_record(struct mail_cache_view *view, uint32_t seq,
        buffer_t *buffer;
        const void *data;
        size_t size, pos;
-       uint32_t nb_size;
+       uint32_t size32;
        int i;
 
        memset(&cache_rec, 0, sizeof(cache_rec));
@@ -37,20 +36,20 @@ mail_cache_compress_record(struct mail_cache_view *view, uint32_t seq,
                        continue;
                }
 
-               nb_size = uint32_to_nbo((uint32_t)size);
+               size32 = (uint32_t)size;
 
                if ((field & MAIL_CACHE_FIXED_MASK) == 0)
-                       buffer_append(buffer, &nb_size, sizeof(nb_size));
+                       buffer_append(buffer, &size32, sizeof(size32));
                buffer_append(buffer, data, size);
-               if ((size & 3) != 0)
-                       buffer_append(buffer, null4, 4 - (size & 3));
+               if ((size32 & 3) != 0)
+                       buffer_append(buffer, null4, 4 - (size32 & 3));
        }
 
        /* now merge all the headers if we have them all */
        if ((orig_cached_fields & mail_cache_header_fields[header_idx]) != 0) {
-               nb_size = 0;
+               size32 = 0;
                pos = buffer_get_used_size(buffer);
-               buffer_append(buffer, &nb_size, sizeof(nb_size));
+               buffer_append(buffer, &size32, sizeof(size32));
 
                for (i = 0; i <= header_idx; i++) {
                        field = mail_cache_header_fields[i];
@@ -58,22 +57,20 @@ mail_cache_compress_record(struct mail_cache_view *view, uint32_t seq,
                                                    &data, &size) && size > 1) {
                                size--; /* terminating \0 */
                                buffer_append(buffer, data, size);
-                               nb_size += size;
+                               size32 += size;
                        }
                }
                buffer_append(buffer, null4, 1);
-               nb_size++;
-               if ((nb_size & 3) != 0)
-                       buffer_append(buffer, null4, 4 - (nb_size & 3));
-
-               nb_size = uint32_to_nbo(nb_size);
-               buffer_write(buffer, pos, &nb_size, sizeof(nb_size));
+               size32++;
+               if ((size32 & 3) != 0)
+                       buffer_append(buffer, null4, 4 - (size32 & 3));
+               buffer_write(buffer, pos, &size32, sizeof(size32));
 
                cached_fields |= MAIL_CACHE_HEADERS1;
        }
 
        cache_rec.fields = cached_fields;
-       cache_rec.size = uint32_to_nbo(buffer_get_used_size(buffer));
+       cache_rec.size = buffer_get_used_size(buffer);
        buffer_write(buffer, 0, &cache_rec, sizeof(cache_rec));
 
        data = buffer_get_data(buffer, &size);
@@ -93,7 +90,7 @@ mail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd)
        enum mail_cache_field keep_fields, temp_fields;
        enum mail_cache_field cached_fields, new_fields;
        const char *str;
-       uint32_t size, nb_size, message_count, seq, first_new_seq;
+       uint32_t size, size32, message_count, seq, first_new_seq;
        uoff_t offset;
        int i, header_idx, ret;
 
@@ -118,6 +115,7 @@ mail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd)
        output = o_stream_create_file(fd, default_pool, 0, FALSE);
 
        memset(&hdr, 0, sizeof(hdr));
+       hdr.version = MAIL_CACHE_VERSION;
        hdr.indexid = idx_hdr->indexid;
        hdr.file_seq = idx_hdr->cache_file_seq + 1;
 
@@ -157,13 +155,11 @@ mail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd)
                        mail_cache_uint32_to_offset(output->offset);
                header_idx = i;
 
-               size = strlen(str) + 1;
-               nb_size = uint32_to_nbo(size);
-
-               o_stream_send(output, &nb_size, sizeof(nb_size));
-               o_stream_send(output, str, size);
-               if ((size & 3) != 0)
-                       o_stream_send(output, null4, 4 - (size & 3));
+               size32 = strlen(str) + 1;
+               o_stream_send(output, &size32, sizeof(size32));
+               o_stream_send(output, str, size32);
+               if ((size32 & 3) != 0)
+                       o_stream_send(output, null4, 4 - (size32 & 3));
        }
 
        mail_index_reset_cache(t, hdr.file_seq);
@@ -183,14 +179,15 @@ mail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd)
                }
 
                if (keep_fields == cached_fields &&
-                   mail_cache_offset_to_uint32(cache_rec->next_offset) == 0) {
+                   cache_rec->prev_offset == 0) {
                        /* just one unmodified block, save it */
-                       size = nbo_to_uint32(cache_rec->size);
                         mail_index_update_cache(t, seq, output->offset);
-                       o_stream_send(output, cache_rec, size);
+                       o_stream_send(output, cache_rec, cache_rec->size);
 
-                       if ((size & 3) != 0)
-                               o_stream_send(output, null4, 4 - (size & 3));
+                       if ((cache_rec->size & 3) != 0) {
+                               o_stream_send(output, null4,
+                                             4 - (cache_rec->size & 3));
+                       }
                } else {
                        /* a) dropping fields
                           b) multiple blocks, sort them into buffer */
@@ -205,7 +202,7 @@ mail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd)
                        t_pop();
                }
        }
-       hdr.used_file_size = uint32_to_nbo(output->offset);
+       hdr.used_file_size = output->offset;
 
        o_stream_seek(output, 0);
        o_stream_send(output, &hdr, sizeof(hdr));
@@ -259,6 +256,8 @@ int mail_cache_compress(struct mail_cache *cache, struct mail_index_view *view)
                return -1;
        }
 
+       // FIXME: check that cache file was just recreated
+
        ret = 0;
        if (mail_cache_copy(cache, view, fd) < 0) {
                (void)file_dotlock_delete(cache->filepath, NULL, fd);
@@ -283,10 +282,8 @@ int mail_cache_compress(struct mail_cache *cache, struct mail_index_view *view)
        memset(cache->split_offsets, 0, sizeof(cache->split_offsets));
        memset(cache->split_headers, 0, sizeof(cache->split_headers));
 
-       if (locked) {
-               if (mail_cache_unlock(cache) < 0)
-                       return -1;
-       }
+       if (locked)
+               mail_cache_unlock(cache);
 
        if (ret == 0)
                 cache->need_compress = FALSE;
index ddd81c71d97f043ca057942dd0f510f07d1684a2..4d639d9ee544e08c79a04109da564fbb572db04e 100644 (file)
@@ -2,7 +2,6 @@
 
 #include "lib.h"
 #include "buffer.h"
-#include "byteorder.h"
 #include "mail-cache-private.h"
 
 #define CACHE_PREFETCH 1024
@@ -32,7 +31,6 @@ mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx)
 
        buf = cache->mmap_base;
        memcpy(&data_size, buf + offset, sizeof(data_size));
-       data_size = nbo_to_uint32(data_size);
        offset += sizeof(data_size);
 
        if (data_size == 0) {
@@ -116,14 +114,10 @@ const char *const *mail_cache_get_header_fields(struct mail_cache_view *view,
 }
 
 struct mail_cache_record *
-mail_cache_get_record(struct mail_cache *cache, uint32_t offset,
-                     int index_offset)
+mail_cache_get_record(struct mail_cache *cache, uint32_t offset)
 {
        struct mail_cache_record *cache_rec;
-       size_t size;
 
-       if (!index_offset)
-               offset = mail_cache_offset_to_uint32(offset);
        if (offset == 0)
                return NULL;
 
@@ -137,17 +131,16 @@ mail_cache_get_record(struct mail_cache *cache, uint32_t offset,
        }
        cache_rec = CACHE_RECORD(cache, offset);
 
-       size = nbo_to_uint32(cache_rec->size);
-       if (size < sizeof(*cache_rec)) {
+       if (cache_rec->size < sizeof(*cache_rec)) {
                mail_cache_set_corrupted(cache, "invalid record size");
                return NULL;
        }
-       if (size > CACHE_PREFETCH) {
-               if (mail_cache_map(cache, offset, size) < 0)
+       if (cache_rec->size > CACHE_PREFETCH) {
+               if (mail_cache_map(cache, offset, cache_rec->size) < 0)
                        return NULL;
        }
 
-       if (offset + size > cache->mmap_length) {
+       if (offset + cache_rec->size > cache->mmap_length) {
                mail_cache_set_corrupted(cache, "record points outside file");
                return NULL;
        }
@@ -195,7 +188,7 @@ mail_cache_lookup(struct mail_cache_view *view, uint32_t seq,
        if (mail_cache_lookup_offset(view, seq, &offset, FALSE) <= 0)
                return NULL;
 
-       return mail_cache_get_record(view->cache, offset, TRUE);
+       return mail_cache_get_record(view->cache, offset);
 }
 
 enum mail_cache_field
@@ -208,26 +201,22 @@ mail_cache_get_fields(struct mail_cache_view *view, uint32_t seq)
        while (cache_rec != NULL) {
                fields |= cache_rec->fields;
                cache_rec = mail_cache_get_record(view->cache,
-                                                 cache_rec->next_offset,
-                                                 FALSE);
+                                                 cache_rec->prev_offset);
        }
 
        return fields;
 }
 
 static int cache_get_field(struct mail_cache *cache,
-                          struct mail_cache_record *cache_rec,
+                          const struct mail_cache_record *cache_rec,
                           enum mail_cache_field field,
                           const void **data_r, size_t *size_r)
 {
-       unsigned char *buf;
        unsigned int mask;
-       uint32_t rec_size, data_size;
-       size_t offset, next_offset;
+       uint32_t data_size;
+       size_t offset, prev_offset;
        int i;
 
-       rec_size = nbo_to_uint32(cache_rec->size);
-       buf = (unsigned char *) cache_rec;
        offset = sizeof(*cache_rec);
 
        for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
@@ -236,7 +225,7 @@ static int cache_get_field(struct mail_cache *cache,
 
                /* all records are at least 32bit. we have to check this
                   before getting data_size. */
-               if (offset + sizeof(uint32_t) > rec_size) {
+               if (offset + sizeof(uint32_t) > cache_rec->size) {
                        mail_cache_set_corrupted(cache,
                                "Record continues outside it's allocated size");
                        return FALSE;
@@ -245,13 +234,13 @@ static int cache_get_field(struct mail_cache *cache,
                if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
                        data_size = mail_cache_field_sizes[i];
                else {
-                       memcpy(&data_size, buf + offset, sizeof(data_size));
-                       data_size = nbo_to_uint32(data_size);
+                       memcpy(&data_size, CONST_PTR_OFFSET(cache_rec, offset),
+                              sizeof(data_size));
                        offset += sizeof(data_size);
                }
 
-               next_offset = offset + ((data_size + 3) & ~3);
-               if (next_offset > rec_size) {
+               prev_offset = offset + ((data_size + 3) & ~3);
+               if (prev_offset > cache_rec->size) {
                        mail_cache_set_corrupted(cache,
                                "Record continues outside it's allocated size");
                        return FALSE;
@@ -263,11 +252,11 @@ static int cache_get_field(struct mail_cache *cache,
                                                         "Field size is 0");
                                return FALSE;
                        }
-                       *data_r = buf + offset;
+                       *data_r = CONST_PTR_OFFSET(cache_rec, offset);
                        *size_r = data_size;
                        return TRUE;
                }
-               offset = next_offset;
+               offset = prev_offset;
        }
 
        i_unreached();
@@ -289,8 +278,7 @@ int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq,
                                               data_r, size_r);
                }
                cache_rec = mail_cache_get_record(view->cache,
-                                                 cache_rec->next_offset,
-                                                 FALSE);
+                                                 cache_rec->prev_offset);
        }
 
        return FALSE;
@@ -308,7 +296,7 @@ mail_cache_lookup_string_field(struct mail_cache_view *view, uint32_t seq,
        if (!mail_cache_lookup_field(view, seq, field, &data, &size))
                return NULL;
 
-       if (((const char *) data)[size-1] != '\0') {
+       if (((const char *)data)[size-1] != '\0') {
                mail_cache_set_corrupted(view->cache,
                        "String field %x doesn't end with NUL", field);
                return NULL;
index 2cf9552e7c30cd3fc0f98830b9f1fbce8877db79..662261ec14a76799c12414eae1fc646e8f4747e7 100644 (file)
@@ -4,6 +4,8 @@
 #include "mail-index-private.h"
 #include "mail-cache.h"
 
+#define MAIL_CACHE_VERSION 1
+
 /* Never compress the file if it's smaller than this */
 #define COMPRESS_MIN_SIZE (1024*50)
 
@@ -43,6 +45,11 @@ enum mail_cache_decision_type {
 };
 
 struct mail_cache_header {
+       /* version is increased only when you can't have backwards
+          compatibility. */
+       uint8_t version;
+       uint8_t unused[3];
+
        uint32_t indexid;
        uint32_t file_seq;
 
@@ -59,7 +66,7 @@ struct mail_cache_header {
 
 struct mail_cache_record {
        uint32_t fields; /* enum mail_cache_field */
-       uint32_t next_offset;
+       uint32_t prev_offset;
        uint32_t size; /* full record size, including this header */
 };
 
@@ -109,8 +116,7 @@ const char *const *
 mail_cache_split_header(struct mail_cache *cache, const char *header);
 
 struct mail_cache_record *
-mail_cache_get_record(struct mail_cache *cache, uint32_t offset,
-                     int index_offset);
+mail_cache_get_record(struct mail_cache *cache, uint32_t offset);
 
 int mail_cache_lookup_offset(struct mail_cache_view *view, uint32_t seq,
                             uint32_t *offset, int skip_expunged);
@@ -126,6 +132,9 @@ int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size);
 void mail_cache_file_close(struct mail_cache *cache);
 int mail_cache_reopen(struct mail_cache *cache);
 
+int mail_cache_link(struct mail_cache *cache, uint32_t old_offset,
+                   uint32_t new_offset);
+
 void mail_cache_handle_decisions(struct mail_cache_view *view, uint32_t seq,
                                 enum mail_cache_field field);
 
index 210358b72b89ebfc7919b62ab54d431e985ce3d6..21377239c2b04f7dc040179f01c1211af22414bd 100644 (file)
@@ -2,7 +2,6 @@
 
 #include "lib.h"
 #include "buffer.h"
-#include "byteorder.h"
 #include "file-set-size.h"
 #include "mmap-util.h"
 #include "mail-cache-private.h"
@@ -46,23 +45,19 @@ int mail_cache_transaction_begin(struct mail_cache_view *view, int nonblock,
        ctx->view = view;
        ctx->trans = t;
        ctx->cache_data = buffer_create_dynamic(system_pool, 8192, (size_t)-1);
-       ctx->used_file_size = nbo_to_uint32(ctx->cache->hdr->used_file_size);
+       ctx->used_file_size = ctx->cache->hdr->used_file_size;
 
        view->cache->trans_ctx = ctx;
        *ctx_r = ctx;
        return 1;
 }
 
-int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx)
+void mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx)
 {
-       int ret = 0;
-
        i_assert(ctx->cache->trans_ctx != NULL);
 
        (void)mail_cache_transaction_rollback(ctx);
-
-       if (mail_cache_unlock(ctx->cache) < 0)
-               ret = -1;
+       mail_cache_unlock(ctx->cache);
 
        ctx->cache->trans_ctx = NULL;
 
@@ -70,7 +65,6 @@ int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx)
                buffer_free(ctx->cache_marks);
        buffer_free(ctx->cache_data);
        i_free(ctx);
-       return ret;
 }
 
 static void mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
@@ -116,7 +110,6 @@ static int write_mark_updates(struct mail_cache *cache)
 static int commit_all_changes(struct mail_cache_transaction_ctx *ctx)
 {
        struct mail_cache *cache = ctx->cache;
-       uint32_t cont;
 
        /* write everything to disk */
        if (msync(cache->mmap_base, cache->mmap_length, MS_SYNC) < 0) {
@@ -138,18 +131,17 @@ static int commit_all_changes(struct mail_cache_transaction_ctx *ctx)
                return -1;
 
        /* update continued records count */
-       cont = nbo_to_uint32(cache->hdr->continued_record_count);
-       cont += buffer_get_used_size(ctx->cache_marks) /
+        cache->hdr->continued_record_count +=
+               buffer_get_used_size(ctx->cache_marks) /
                (sizeof(uint32_t) * 2);
 
-       if (cont * 100 / cache->index->hdr->messages_count >=
+       if (cache->hdr->continued_record_count * 100 /
+           cache->index->hdr->messages_count >=
            COMPRESS_CONTINUED_PERCENTAGE &&
            ctx->used_file_size >= COMPRESS_MIN_SIZE) {
                /* too many continued rows, compress */
                cache->need_compress = TRUE;
        }
-
-       cache->hdr->continued_record_count = uint32_to_nbo(cont);
        return 0;
 }
 
@@ -218,7 +210,6 @@ static uint32_t mail_cache_append_space(struct mail_cache_transaction_ctx *ctx,
 static int mail_cache_write(struct mail_cache_transaction_ctx *ctx)
 {
        struct mail_cache *cache = ctx->cache;
-       struct mail_cache_record *cache_rec;
        uint32_t offset, write_offset;
        const void *buf;
        size_t size, buf_size;
@@ -226,8 +217,8 @@ static int mail_cache_write(struct mail_cache_transaction_ctx *ctx)
 
        buf = buffer_get_data(ctx->cache_data, &buf_size);
 
-       size = sizeof(*cache_rec) + buf_size;
-       ctx->cache_rec.size = uint32_to_nbo(size);
+       size = sizeof(ctx->cache_rec) + buf_size;
+       ctx->cache_rec.size = size;
 
         ret = mail_cache_lookup_offset(ctx->view, ctx->prev_seq, &offset, TRUE);
        if (ret < 0)
@@ -240,27 +231,10 @@ static int mail_cache_write(struct mail_cache_transaction_ctx *ctx)
                if (write_offset == 0)
                        return -1;
 
-               cache_rec = mail_cache_get_record(cache, offset, TRUE);
-               if (cache_rec == NULL) {
-                       /* first cache record - update offset in index file */
-                       mail_index_update_cache(ctx->trans, ctx->prev_seq,
-                                               write_offset);
-               } else {
-                       /* find offset to last cache record */
-                       for (;;) {
-                               cache_rec = mail_cache_get_record(cache, offset,
-                                                                 FALSE);
-                               if (cache_rec == NULL)
-                                       break;
-                               offset = cache_rec->next_offset;
-                       }
-
-                       /* mark next_offset to be updated later */
-                       offset = mail_cache_offset_to_uint32(offset) +
-                               offsetof(struct mail_cache_record, next_offset);
-                       mark_update(&ctx->cache_marks, offset,
-                                   mail_cache_uint32_to_offset(write_offset));
-               }
+               /* write the offset to index file. this record's prev_offset
+                  is updated to point to old cache record when index is
+                  being synced. */
+               mail_index_update_cache(ctx->trans, ctx->prev_seq, write_offset);
 
                memcpy((char *) cache->mmap_base + write_offset,
                       &ctx->cache_rec, sizeof(ctx->cache_rec));
@@ -291,7 +265,7 @@ int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx)
                        return -1;
        }
 
-       ctx->cache->hdr->used_file_size = uint32_to_nbo(ctx->used_file_size);
+       ctx->cache->hdr->used_file_size = ctx->used_file_size;
 
        if (commit_all_changes(ctx) < 0)
                ret = -1;
@@ -312,7 +286,7 @@ void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx)
 
        /* no need to actually modify the file - we just didn't update
           used_file_size */
-       ctx->used_file_size = nbo_to_uint32(cache->hdr->used_file_size);
+       ctx->used_file_size = cache->hdr->used_file_size;
 
        /* make sure we don't cache the headers */
        for (i = 0; i < ctx->next_unused_header_lowwater; i++) {
@@ -378,11 +352,10 @@ int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx,
 
        offset = mail_cache_append_space(ctx, size + sizeof(uint32_t));
        if (offset != 0) {
-               memcpy((char *) cache->mmap_base + offset + sizeof(uint32_t),
+               memcpy(PTR_OFFSET(cache->mmap_base, offset + sizeof(uint32_t)),
                       header_str, size);
 
-               size = uint32_to_nbo(size);
-               memcpy((char *) cache->mmap_base + offset,
+               memcpy(PTR_OFFSET(cache->mmap_base, offset),
                       &size, sizeof(uint32_t));
 
                /* update cached headers */
@@ -428,7 +401,6 @@ static size_t get_insert_offset(struct mail_cache_transaction_ctx *ctx,
                        else {
                                memcpy(&data_size, buf + offset,
                                       sizeof(data_size));
-                               data_size = nbo_to_uint32(data_size);
                                offset += sizeof(data_size);
                        }
                        offset += (data_size + 3) & ~3;
@@ -456,7 +428,7 @@ int mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
                   enum mail_cache_field field,
                   const void *data, size_t data_size)
 {
-       uint32_t nb_data_size;
+       uint32_t data_size32;
        size_t full_size, offset;
        unsigned char *buf;
        int field_num;
@@ -464,7 +436,7 @@ int mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
        i_assert(data_size > 0);
        i_assert(data_size < (uint32_t)-1);
 
-       nb_data_size = uint32_to_nbo((uint32_t)data_size);
+       data_size32 = (uint32_t)data_size;
 
        if ((field & MAIL_CACHE_FIXED_MASK) != 0) {
                field_num = get_field_num(field);
@@ -484,7 +456,7 @@ int mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
 
        full_size = (data_size + 3) & ~3;
        if ((field & MAIL_CACHE_FIXED_MASK) == 0)
-               full_size += sizeof(nb_data_size);
+               full_size += sizeof(data_size32);
 
        /* fields must be ordered. find where to insert it. */
        if (field > ctx->cache_rec.fields)
@@ -500,8 +472,8 @@ int mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
 
        /* @UNSAFE */
        if ((field & MAIL_CACHE_FIXED_MASK) == 0) {
-               memcpy(buf, &nb_data_size, sizeof(nb_data_size));
-               buf += sizeof(nb_data_size);
+               memcpy(buf, &data_size32, sizeof(data_size32));
+               buf += sizeof(data_size32);
        }
        memcpy(buf, data, data_size); buf += data_size;
        if ((data_size & 3) != 0)
@@ -533,12 +505,12 @@ int mail_cache_delete(struct mail_cache_transaction_ctx *ctx, uint32_t seq)
           the data. also it's actually useful as some index views are still
           able to ask cached data from messages that have already been
           expunged. */
-       deleted_space = nbo_to_uint32(cache->hdr->deleted_space);
+       deleted_space = cache->hdr->deleted_space;
 
        do {
-               deleted_space -= nbo_to_uint32(cache_rec->size);
-               cache_rec = mail_cache_get_record(cache, cache_rec->next_offset,
-                                                 FALSE);
+               deleted_space += cache_rec->size;
+               cache_rec =
+                       mail_cache_get_record(cache, cache_rec->prev_offset);
        } while (cache_rec != NULL);
 
        /* see if we've reached the max. deleted space in file */
@@ -547,7 +519,7 @@ int mail_cache_delete(struct mail_cache_transaction_ctx *ctx, uint32_t seq)
            ctx->used_file_size >= COMPRESS_MIN_SIZE)
                cache->need_compress = TRUE;
 
-       cache->hdr->deleted_space = uint32_to_nbo(deleted_space);
+       cache->hdr->deleted_space = deleted_space;
        return 0;
 }
 
@@ -580,3 +552,22 @@ int mail_cache_update_record_flags(struct mail_cache_view *view, uint32_t seq,
 {
        return -1;
 }
+
+int mail_cache_link(struct mail_cache *cache, uint32_t old_offset,
+                   uint32_t new_offset)
+{
+       struct mail_cache_record *cache_rec;
+
+       i_assert(cache->locks > 0);
+
+       if (mail_cache_map(cache, new_offset, sizeof(*cache_rec)) < 0)
+               return -1;
+
+       if (new_offset + sizeof(*cache_rec) > cache->mmap_length) {
+               mail_cache_set_corrupted(cache, "record points outside file");
+               return -1;
+       }
+       cache_rec = CACHE_RECORD(cache, new_offset);
+       cache_rec->prev_offset = old_offset;
+       return 0;
+}
index e04e2f76887a5b71dc02b1460dfc291b15d6b2f7..a0389c897061d06b6c787a0ab16d99fcaa2513b0 100644 (file)
@@ -1,7 +1,6 @@
 /* Copyright (C) 2003-2004 Timo Sirainen */
 
 #include "lib.h"
-#include "byteorder.h"
 #include "file-lock.h"
 #include "file-set-size.h"
 #include "mmap-util.h"
@@ -156,7 +155,6 @@ int mail_cache_reopen(struct mail_cache *cache)
 static int mmap_verify_header(struct mail_cache *cache)
 {
        struct mail_cache_header *hdr;
-       uint32_t used_file_size;
 
        /* check that the header is still ok */
        if (cache->mmap_length < sizeof(struct mail_cache_header)) {
@@ -165,6 +163,11 @@ static int mmap_verify_header(struct mail_cache *cache)
        }
        cache->hdr = hdr = cache->mmap_base;
 
+       if (cache->hdr->version != MAIL_CACHE_VERSION) {
+               /* version changed - upgrade silently */
+               return FALSE;
+       }
+
        if (cache->hdr->indexid != cache->index->indexid) {
                /* index id changed */
                mail_cache_set_corrupted(cache, "indexid changed");
@@ -180,17 +183,16 @@ static int mmap_verify_header(struct mail_cache *cache)
        if (cache->locks == 0)
                return TRUE;
 
-       used_file_size = nbo_to_uint32(hdr->used_file_size);
-       if (used_file_size < sizeof(struct mail_cache_header)) {
+       if (hdr->used_file_size < sizeof(struct mail_cache_header)) {
                mail_cache_set_corrupted(cache, "used_file_size too small");
                return FALSE;
        }
-       if ((used_file_size % sizeof(uint32_t)) != 0) {
+       if ((hdr->used_file_size % sizeof(uint32_t)) != 0) {
                mail_cache_set_corrupted(cache, "used_file_size not aligned");
                return FALSE;
        }
 
-       if (used_file_size > cache->mmap_length) {
+       if (hdr->used_file_size > cache->mmap_length) {
                mail_cache_set_corrupted(cache, "used_file_size too large");
                return FALSE;
        }
@@ -354,17 +356,13 @@ int mail_cache_lock(struct mail_cache *cache, int nonblock)
        return ret;
 }
 
-int mail_cache_unlock(struct mail_cache *cache)
+void mail_cache_unlock(struct mail_cache *cache)
 {
        if (--cache->locks > 0)
-               return 0;
+               return;
 
-       if (file_wait_lock(cache->fd, F_UNLCK) <= 0) {
+       if (file_wait_lock(cache->fd, F_UNLCK) <= 0)
                mail_cache_set_syscall_error(cache, "file_wait_lock(F_UNLCK)");
-               return -1;
-       }
-
-       return 0;
 }
 
 int mail_cache_is_locked(struct mail_cache *cache)
index 9b889de5a29705d740b1b8686f5c161a35084893..8b81ea3079d573269ead28c98ae00809fb6ae484 100644 (file)
@@ -86,7 +86,7 @@ int mail_cache_compress(struct mail_cache *cache, struct mail_index_view *view);
 /* Explicitly lock the cache file. Returns -1 if error, 1 if ok, 0 if we
    couldn't lock */
 int mail_cache_lock(struct mail_cache *cache, int nonblock);
-int mail_cache_unlock(struct mail_cache *cache);
+void mail_cache_unlock(struct mail_cache *cache);
 
 /* Returns TRUE if cache file is locked. */
 int mail_cache_is_locked(struct mail_cache *cache);
@@ -106,8 +106,7 @@ int mail_cache_transaction_begin(struct mail_cache_view *view, int nonblock,
 int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx);
 void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx);
 
-/* Should be called only by mail_transaction_commit/rollback: */
-int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx);
+void mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx);
 
 /* Return NULL-terminated list of headers for given index, or NULL if
    header index isn't used. */
index e3260c4a1f3951a7ed78ca1d3615869b8228021e..1814a53ab0d872b104609e1e75e5e36155f3e36d 100644 (file)
@@ -22,6 +22,7 @@ struct mail_index_sync_ctx {
        unsigned int lock_id;
 
        unsigned int sync_appends:1;
+       unsigned int cache_locked:1;
 };
 
 extern struct mail_transaction_map_functions mail_index_map_sync_funcs;
index 3f9443c6732e1363fa320f1d6897dc86b9ec9c92..bf135d8a1a7f2c2451eee2290335c13fac2db372 100644 (file)
@@ -9,6 +9,7 @@
 #include "mail-index-sync-private.h"
 #include "mail-transaction-log.h"
 #include "mail-transaction-util.h"
+#include "mail-cache-private.h"
 
 #include <time.h>
 
@@ -58,7 +59,8 @@ mail_index_header_update_lowwaters(struct mail_index_header *hdr,
 
 static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
 {
-       struct mail_index_view *view = context;
+        struct mail_index_sync_ctx *sync_ctx = context;
+       struct mail_index_view *view = sync_ctx->view;
        struct mail_index_map *map = view->map;
        struct mail_index_header *hdr = &map->hdr_copy;
        struct mail_index_record *rec;
@@ -98,7 +100,8 @@ static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
 static int sync_append(const struct mail_transaction_append_header *hdr,
                       const struct mail_index_record *rec, void *context)
 {
-       struct mail_index_view *view = context;
+        struct mail_index_sync_ctx *sync_ctx = context;
+       struct mail_index_view *view = sync_ctx->view;
        struct mail_index_map *map = view->map;
        void *dest;
 
@@ -139,7 +142,8 @@ static int sync_append(const struct mail_transaction_append_header *hdr,
 static int sync_flag_update(const struct mail_transaction_flag_update *u,
                            void *context)
 {
-        struct mail_index_view *view = context;
+        struct mail_index_sync_ctx *sync_ctx = context;
+       struct mail_index_view *view = sync_ctx->view;
        struct mail_index_record *rec;
        struct mail_index_header *hdr;
        uint8_t flag_mask, old_flags;
@@ -188,7 +192,8 @@ static int sync_flag_update(const struct mail_transaction_flag_update *u,
 static int sync_cache_reset(const struct mail_transaction_cache_reset *u,
                            void *context)
 {
-       struct mail_index_view *view = context;
+        struct mail_index_sync_ctx *sync_ctx = context;
+       struct mail_index_view *view = sync_ctx->view;
        uint32_t i;
 
        view->map->hdr_copy.cache_file_seq = u->new_file_seq;
@@ -201,7 +206,9 @@ static int sync_cache_reset(const struct mail_transaction_cache_reset *u,
 static int sync_cache_update(const struct mail_transaction_cache_update *u,
                             void *context)
 {
-       struct mail_index_view *view = context;
+        struct mail_index_sync_ctx *sync_ctx = context;
+       struct mail_index_view *view = sync_ctx->view;
+       struct mail_index_record *rec;
        uint32_t seq;
        int ret;
 
@@ -209,10 +216,25 @@ static int sync_cache_update(const struct mail_transaction_cache_update *u,
                                          &seq, &seq);
        i_assert(ret == 0);
 
-       if (seq != 0) {
-               MAIL_INDEX_MAP_IDX(view->map, seq-1)->cache_offset =
-                       u->cache_offset;
+       if (seq == 0) {
+               /* already expunged */
+               return 1;
        }
+
+       rec = MAIL_INDEX_MAP_IDX(view->map, seq-1);
+       if (rec->cache_offset != 0) {
+               /* we'll need to link the old and new cache records */
+               if (!sync_ctx->cache_locked) {
+                       if (mail_cache_lock(view->index->cache, FALSE) <= 0)
+                               return -1;
+                       sync_ctx->cache_locked = TRUE;
+               }
+
+               if (mail_cache_link(view->index->cache,
+                                   rec->cache_offset, u->cache_offset) < 0)
+                       return -1;
+       }
+       rec->cache_offset = u->cache_offset;
        return 1;
 }
 
@@ -423,12 +445,17 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
 
                if (mail_transaction_map(index, hdr, data,
                                         &mail_index_map_sync_funcs,
-                                        view) < 0) {
+                                        sync_ctx) < 0) {
                        ret = -1;
                        break;
                }
        }
 
+       if (sync_ctx->cache_locked) {
+               mail_cache_unlock(index->cache);
+               sync_ctx->cache_locked = FALSE;
+       }
+
        if (ret < 0) {
                mail_index_view_unlock(view);
                return -1;
index 76a4dd17ca688798cf057edd9f28a69cf4a3b3ef..83805a791b2e3f302fabc64a918dc18db59d5f1b 100644 (file)
@@ -792,7 +792,9 @@ void index_mail_headers_close(struct index_mail *mail)
        len = str_len(mail->data.header_data) -
                data->header_data_uncached_offset;
 
-       mail_cache_add(mail->trans->cache_trans, data->seq,
-                      mail_cache_header_fields[idx], str, len+1);
+       if (len != 0) {
+               mail_cache_add(mail->trans->cache_trans, data->seq,
+                              mail_cache_header_fields[idx], str, len+1);
+       }
        data->header_save = FALSE;
 }
index 159be2b82e62dd540d1f31f5301d2a99f770a3f1..616e19b860b1e7c633a9bc860024bdeba19dcba3 100644 (file)
@@ -16,7 +16,7 @@ void index_transaction_init(struct index_transaction_context *t,
 static void index_transaction_free(struct index_transaction_context *t)
 {
        if (t->cache_trans != NULL)
-               (void)mail_cache_transaction_end(t->cache_trans);
+               mail_cache_transaction_end(t->cache_trans);
 
        mail_cache_view_close(t->cache_view);
        mail_index_view_close(t->trans_view);