From: Timo Sirainen Date: Sun, 4 Jul 2004 14:26:22 +0000 (+0300) Subject: Caching fixes and optimizations. Removed all network byte ordering code - X-Git-Tag: 1.1.alpha1~3830 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a40d26f83af808a0ea1e212c001d682a96d870b0;p=thirdparty%2Fdovecot%2Fcore.git Caching fixes and optimizations. Removed all network byte ordering code - it's not worth the trouble and would require massive changes in indexing code as well to be useful. Changed next_offset to prev_offset which is updated while syncing index. --HG-- branch : HEAD --- diff --git a/src/lib-index/mail-cache-compress.c b/src/lib-index/mail-cache-compress.c index 555ff7ffd8..5736ae3ee2 100644 --- a/src/lib-index/mail-cache-compress.c +++ b/src/lib-index/mail-cache-compress.c @@ -2,7 +2,6 @@ #include "lib.h" #include "buffer.h" -#include "byteorder.h" #include "ostream.h" #include "file-set-size.h" #include "mail-cache-private.h" @@ -19,7 +18,7 @@ mail_cache_compress_record(struct mail_cache_view *view, uint32_t seq, buffer_t *buffer; const void *data; size_t size, pos; - uint32_t nb_size; + uint32_t size32; int i; memset(&cache_rec, 0, sizeof(cache_rec)); @@ -37,20 +36,20 @@ mail_cache_compress_record(struct mail_cache_view *view, uint32_t seq, continue; } - nb_size = uint32_to_nbo((uint32_t)size); + size32 = (uint32_t)size; if ((field & MAIL_CACHE_FIXED_MASK) == 0) - buffer_append(buffer, &nb_size, sizeof(nb_size)); + buffer_append(buffer, &size32, sizeof(size32)); buffer_append(buffer, data, size); - if ((size & 3) != 0) - buffer_append(buffer, null4, 4 - (size & 3)); + if ((size32 & 3) != 0) + buffer_append(buffer, null4, 4 - (size32 & 3)); } /* now merge all the headers if we have them all */ if ((orig_cached_fields & mail_cache_header_fields[header_idx]) != 0) { - nb_size = 0; + size32 = 0; pos = buffer_get_used_size(buffer); - buffer_append(buffer, &nb_size, sizeof(nb_size)); + buffer_append(buffer, &size32, sizeof(size32)); for (i = 0; i <= header_idx; i++) { field = mail_cache_header_fields[i]; @@ -58,22 +57,20 @@ mail_cache_compress_record(struct mail_cache_view *view, uint32_t seq, &data, &size) && size > 1) { size--; /* terminating \0 */ buffer_append(buffer, data, size); - nb_size += size; + size32 += size; } } buffer_append(buffer, null4, 1); - nb_size++; - if ((nb_size & 3) != 0) - buffer_append(buffer, null4, 4 - (nb_size & 3)); - - nb_size = uint32_to_nbo(nb_size); - buffer_write(buffer, pos, &nb_size, sizeof(nb_size)); + size32++; + if ((size32 & 3) != 0) + buffer_append(buffer, null4, 4 - (size32 & 3)); + buffer_write(buffer, pos, &size32, sizeof(size32)); cached_fields |= MAIL_CACHE_HEADERS1; } cache_rec.fields = cached_fields; - cache_rec.size = uint32_to_nbo(buffer_get_used_size(buffer)); + cache_rec.size = buffer_get_used_size(buffer); buffer_write(buffer, 0, &cache_rec, sizeof(cache_rec)); data = buffer_get_data(buffer, &size); @@ -93,7 +90,7 @@ mail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd) enum mail_cache_field keep_fields, temp_fields; enum mail_cache_field cached_fields, new_fields; const char *str; - uint32_t size, nb_size, message_count, seq, first_new_seq; + uint32_t size, size32, message_count, seq, first_new_seq; uoff_t offset; int i, header_idx, ret; @@ -118,6 +115,7 @@ mail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd) output = o_stream_create_file(fd, default_pool, 0, FALSE); memset(&hdr, 0, sizeof(hdr)); + hdr.version = MAIL_CACHE_VERSION; hdr.indexid = idx_hdr->indexid; hdr.file_seq = idx_hdr->cache_file_seq + 1; @@ -157,13 +155,11 @@ mail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd) mail_cache_uint32_to_offset(output->offset); header_idx = i; - size = strlen(str) + 1; - nb_size = uint32_to_nbo(size); - - o_stream_send(output, &nb_size, sizeof(nb_size)); - o_stream_send(output, str, size); - if ((size & 3) != 0) - o_stream_send(output, null4, 4 - (size & 3)); + size32 = strlen(str) + 1; + o_stream_send(output, &size32, sizeof(size32)); + o_stream_send(output, str, size32); + if ((size32 & 3) != 0) + o_stream_send(output, null4, 4 - (size32 & 3)); } mail_index_reset_cache(t, hdr.file_seq); @@ -183,14 +179,15 @@ mail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd) } if (keep_fields == cached_fields && - mail_cache_offset_to_uint32(cache_rec->next_offset) == 0) { + cache_rec->prev_offset == 0) { /* just one unmodified block, save it */ - size = nbo_to_uint32(cache_rec->size); mail_index_update_cache(t, seq, output->offset); - o_stream_send(output, cache_rec, size); + o_stream_send(output, cache_rec, cache_rec->size); - if ((size & 3) != 0) - o_stream_send(output, null4, 4 - (size & 3)); + if ((cache_rec->size & 3) != 0) { + o_stream_send(output, null4, + 4 - (cache_rec->size & 3)); + } } else { /* a) dropping fields b) multiple blocks, sort them into buffer */ @@ -205,7 +202,7 @@ mail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd) t_pop(); } } - hdr.used_file_size = uint32_to_nbo(output->offset); + hdr.used_file_size = output->offset; o_stream_seek(output, 0); o_stream_send(output, &hdr, sizeof(hdr)); @@ -259,6 +256,8 @@ int mail_cache_compress(struct mail_cache *cache, struct mail_index_view *view) return -1; } + // FIXME: check that cache file was just recreated + ret = 0; if (mail_cache_copy(cache, view, fd) < 0) { (void)file_dotlock_delete(cache->filepath, NULL, fd); @@ -283,10 +282,8 @@ int mail_cache_compress(struct mail_cache *cache, struct mail_index_view *view) memset(cache->split_offsets, 0, sizeof(cache->split_offsets)); memset(cache->split_headers, 0, sizeof(cache->split_headers)); - if (locked) { - if (mail_cache_unlock(cache) < 0) - return -1; - } + if (locked) + mail_cache_unlock(cache); if (ret == 0) cache->need_compress = FALSE; diff --git a/src/lib-index/mail-cache-lookup.c b/src/lib-index/mail-cache-lookup.c index ddd81c71d9..4d639d9ee5 100644 --- a/src/lib-index/mail-cache-lookup.c +++ b/src/lib-index/mail-cache-lookup.c @@ -2,7 +2,6 @@ #include "lib.h" #include "buffer.h" -#include "byteorder.h" #include "mail-cache-private.h" #define CACHE_PREFETCH 1024 @@ -32,7 +31,6 @@ mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx) buf = cache->mmap_base; memcpy(&data_size, buf + offset, sizeof(data_size)); - data_size = nbo_to_uint32(data_size); offset += sizeof(data_size); if (data_size == 0) { @@ -116,14 +114,10 @@ const char *const *mail_cache_get_header_fields(struct mail_cache_view *view, } struct mail_cache_record * -mail_cache_get_record(struct mail_cache *cache, uint32_t offset, - int index_offset) +mail_cache_get_record(struct mail_cache *cache, uint32_t offset) { struct mail_cache_record *cache_rec; - size_t size; - if (!index_offset) - offset = mail_cache_offset_to_uint32(offset); if (offset == 0) return NULL; @@ -137,17 +131,16 @@ mail_cache_get_record(struct mail_cache *cache, uint32_t offset, } cache_rec = CACHE_RECORD(cache, offset); - size = nbo_to_uint32(cache_rec->size); - if (size < sizeof(*cache_rec)) { + if (cache_rec->size < sizeof(*cache_rec)) { mail_cache_set_corrupted(cache, "invalid record size"); return NULL; } - if (size > CACHE_PREFETCH) { - if (mail_cache_map(cache, offset, size) < 0) + if (cache_rec->size > CACHE_PREFETCH) { + if (mail_cache_map(cache, offset, cache_rec->size) < 0) return NULL; } - if (offset + size > cache->mmap_length) { + if (offset + cache_rec->size > cache->mmap_length) { mail_cache_set_corrupted(cache, "record points outside file"); return NULL; } @@ -195,7 +188,7 @@ mail_cache_lookup(struct mail_cache_view *view, uint32_t seq, if (mail_cache_lookup_offset(view, seq, &offset, FALSE) <= 0) return NULL; - return mail_cache_get_record(view->cache, offset, TRUE); + return mail_cache_get_record(view->cache, offset); } enum mail_cache_field @@ -208,26 +201,22 @@ mail_cache_get_fields(struct mail_cache_view *view, uint32_t seq) while (cache_rec != NULL) { fields |= cache_rec->fields; cache_rec = mail_cache_get_record(view->cache, - cache_rec->next_offset, - FALSE); + cache_rec->prev_offset); } return fields; } static int cache_get_field(struct mail_cache *cache, - struct mail_cache_record *cache_rec, + const struct mail_cache_record *cache_rec, enum mail_cache_field field, const void **data_r, size_t *size_r) { - unsigned char *buf; unsigned int mask; - uint32_t rec_size, data_size; - size_t offset, next_offset; + uint32_t data_size; + size_t offset, prev_offset; int i; - rec_size = nbo_to_uint32(cache_rec->size); - buf = (unsigned char *) cache_rec; offset = sizeof(*cache_rec); for (i = 0, mask = 1; i < 31; i++, mask <<= 1) { @@ -236,7 +225,7 @@ static int cache_get_field(struct mail_cache *cache, /* all records are at least 32bit. we have to check this before getting data_size. */ - if (offset + sizeof(uint32_t) > rec_size) { + if (offset + sizeof(uint32_t) > cache_rec->size) { mail_cache_set_corrupted(cache, "Record continues outside it's allocated size"); return FALSE; @@ -245,13 +234,13 @@ static int cache_get_field(struct mail_cache *cache, if ((mask & MAIL_CACHE_FIXED_MASK) != 0) data_size = mail_cache_field_sizes[i]; else { - memcpy(&data_size, buf + offset, sizeof(data_size)); - data_size = nbo_to_uint32(data_size); + memcpy(&data_size, CONST_PTR_OFFSET(cache_rec, offset), + sizeof(data_size)); offset += sizeof(data_size); } - next_offset = offset + ((data_size + 3) & ~3); - if (next_offset > rec_size) { + prev_offset = offset + ((data_size + 3) & ~3); + if (prev_offset > cache_rec->size) { mail_cache_set_corrupted(cache, "Record continues outside it's allocated size"); return FALSE; @@ -263,11 +252,11 @@ static int cache_get_field(struct mail_cache *cache, "Field size is 0"); return FALSE; } - *data_r = buf + offset; + *data_r = CONST_PTR_OFFSET(cache_rec, offset); *size_r = data_size; return TRUE; } - offset = next_offset; + offset = prev_offset; } i_unreached(); @@ -289,8 +278,7 @@ int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq, data_r, size_r); } cache_rec = mail_cache_get_record(view->cache, - cache_rec->next_offset, - FALSE); + cache_rec->prev_offset); } return FALSE; @@ -308,7 +296,7 @@ mail_cache_lookup_string_field(struct mail_cache_view *view, uint32_t seq, if (!mail_cache_lookup_field(view, seq, field, &data, &size)) return NULL; - if (((const char *) data)[size-1] != '\0') { + if (((const char *)data)[size-1] != '\0') { mail_cache_set_corrupted(view->cache, "String field %x doesn't end with NUL", field); return NULL; diff --git a/src/lib-index/mail-cache-private.h b/src/lib-index/mail-cache-private.h index 2cf9552e7c..662261ec14 100644 --- a/src/lib-index/mail-cache-private.h +++ b/src/lib-index/mail-cache-private.h @@ -4,6 +4,8 @@ #include "mail-index-private.h" #include "mail-cache.h" +#define MAIL_CACHE_VERSION 1 + /* Never compress the file if it's smaller than this */ #define COMPRESS_MIN_SIZE (1024*50) @@ -43,6 +45,11 @@ enum mail_cache_decision_type { }; struct mail_cache_header { + /* version is increased only when you can't have backwards + compatibility. */ + uint8_t version; + uint8_t unused[3]; + uint32_t indexid; uint32_t file_seq; @@ -59,7 +66,7 @@ struct mail_cache_header { struct mail_cache_record { uint32_t fields; /* enum mail_cache_field */ - uint32_t next_offset; + uint32_t prev_offset; uint32_t size; /* full record size, including this header */ }; @@ -109,8 +116,7 @@ const char *const * mail_cache_split_header(struct mail_cache *cache, const char *header); struct mail_cache_record * -mail_cache_get_record(struct mail_cache *cache, uint32_t offset, - int index_offset); +mail_cache_get_record(struct mail_cache *cache, uint32_t offset); int mail_cache_lookup_offset(struct mail_cache_view *view, uint32_t seq, uint32_t *offset, int skip_expunged); @@ -126,6 +132,9 @@ int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size); void mail_cache_file_close(struct mail_cache *cache); int mail_cache_reopen(struct mail_cache *cache); +int mail_cache_link(struct mail_cache *cache, uint32_t old_offset, + uint32_t new_offset); + void mail_cache_handle_decisions(struct mail_cache_view *view, uint32_t seq, enum mail_cache_field field); diff --git a/src/lib-index/mail-cache-transaction.c b/src/lib-index/mail-cache-transaction.c index 210358b72b..21377239c2 100644 --- a/src/lib-index/mail-cache-transaction.c +++ b/src/lib-index/mail-cache-transaction.c @@ -2,7 +2,6 @@ #include "lib.h" #include "buffer.h" -#include "byteorder.h" #include "file-set-size.h" #include "mmap-util.h" #include "mail-cache-private.h" @@ -46,23 +45,19 @@ int mail_cache_transaction_begin(struct mail_cache_view *view, int nonblock, ctx->view = view; ctx->trans = t; ctx->cache_data = buffer_create_dynamic(system_pool, 8192, (size_t)-1); - ctx->used_file_size = nbo_to_uint32(ctx->cache->hdr->used_file_size); + ctx->used_file_size = ctx->cache->hdr->used_file_size; view->cache->trans_ctx = ctx; *ctx_r = ctx; return 1; } -int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx) +void mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx) { - int ret = 0; - i_assert(ctx->cache->trans_ctx != NULL); (void)mail_cache_transaction_rollback(ctx); - - if (mail_cache_unlock(ctx->cache) < 0) - ret = -1; + mail_cache_unlock(ctx->cache); ctx->cache->trans_ctx = NULL; @@ -70,7 +65,6 @@ int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx) buffer_free(ctx->cache_marks); buffer_free(ctx->cache_data); i_free(ctx); - return ret; } static void mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx) @@ -116,7 +110,6 @@ static int write_mark_updates(struct mail_cache *cache) static int commit_all_changes(struct mail_cache_transaction_ctx *ctx) { struct mail_cache *cache = ctx->cache; - uint32_t cont; /* write everything to disk */ if (msync(cache->mmap_base, cache->mmap_length, MS_SYNC) < 0) { @@ -138,18 +131,17 @@ static int commit_all_changes(struct mail_cache_transaction_ctx *ctx) return -1; /* update continued records count */ - cont = nbo_to_uint32(cache->hdr->continued_record_count); - cont += buffer_get_used_size(ctx->cache_marks) / + cache->hdr->continued_record_count += + buffer_get_used_size(ctx->cache_marks) / (sizeof(uint32_t) * 2); - if (cont * 100 / cache->index->hdr->messages_count >= + if (cache->hdr->continued_record_count * 100 / + cache->index->hdr->messages_count >= COMPRESS_CONTINUED_PERCENTAGE && ctx->used_file_size >= COMPRESS_MIN_SIZE) { /* too many continued rows, compress */ cache->need_compress = TRUE; } - - cache->hdr->continued_record_count = uint32_to_nbo(cont); return 0; } @@ -218,7 +210,6 @@ static uint32_t mail_cache_append_space(struct mail_cache_transaction_ctx *ctx, static int mail_cache_write(struct mail_cache_transaction_ctx *ctx) { struct mail_cache *cache = ctx->cache; - struct mail_cache_record *cache_rec; uint32_t offset, write_offset; const void *buf; size_t size, buf_size; @@ -226,8 +217,8 @@ static int mail_cache_write(struct mail_cache_transaction_ctx *ctx) buf = buffer_get_data(ctx->cache_data, &buf_size); - size = sizeof(*cache_rec) + buf_size; - ctx->cache_rec.size = uint32_to_nbo(size); + size = sizeof(ctx->cache_rec) + buf_size; + ctx->cache_rec.size = size; ret = mail_cache_lookup_offset(ctx->view, ctx->prev_seq, &offset, TRUE); if (ret < 0) @@ -240,27 +231,10 @@ static int mail_cache_write(struct mail_cache_transaction_ctx *ctx) if (write_offset == 0) return -1; - cache_rec = mail_cache_get_record(cache, offset, TRUE); - if (cache_rec == NULL) { - /* first cache record - update offset in index file */ - mail_index_update_cache(ctx->trans, ctx->prev_seq, - write_offset); - } else { - /* find offset to last cache record */ - for (;;) { - cache_rec = mail_cache_get_record(cache, offset, - FALSE); - if (cache_rec == NULL) - break; - offset = cache_rec->next_offset; - } - - /* mark next_offset to be updated later */ - offset = mail_cache_offset_to_uint32(offset) + - offsetof(struct mail_cache_record, next_offset); - mark_update(&ctx->cache_marks, offset, - mail_cache_uint32_to_offset(write_offset)); - } + /* write the offset to index file. this record's prev_offset + is updated to point to old cache record when index is + being synced. */ + mail_index_update_cache(ctx->trans, ctx->prev_seq, write_offset); memcpy((char *) cache->mmap_base + write_offset, &ctx->cache_rec, sizeof(ctx->cache_rec)); @@ -291,7 +265,7 @@ int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx) return -1; } - ctx->cache->hdr->used_file_size = uint32_to_nbo(ctx->used_file_size); + ctx->cache->hdr->used_file_size = ctx->used_file_size; if (commit_all_changes(ctx) < 0) ret = -1; @@ -312,7 +286,7 @@ void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx) /* no need to actually modify the file - we just didn't update used_file_size */ - ctx->used_file_size = nbo_to_uint32(cache->hdr->used_file_size); + ctx->used_file_size = cache->hdr->used_file_size; /* make sure we don't cache the headers */ for (i = 0; i < ctx->next_unused_header_lowwater; i++) { @@ -378,11 +352,10 @@ int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx, offset = mail_cache_append_space(ctx, size + sizeof(uint32_t)); if (offset != 0) { - memcpy((char *) cache->mmap_base + offset + sizeof(uint32_t), + memcpy(PTR_OFFSET(cache->mmap_base, offset + sizeof(uint32_t)), header_str, size); - size = uint32_to_nbo(size); - memcpy((char *) cache->mmap_base + offset, + memcpy(PTR_OFFSET(cache->mmap_base, offset), &size, sizeof(uint32_t)); /* update cached headers */ @@ -428,7 +401,6 @@ static size_t get_insert_offset(struct mail_cache_transaction_ctx *ctx, else { memcpy(&data_size, buf + offset, sizeof(data_size)); - data_size = nbo_to_uint32(data_size); offset += sizeof(data_size); } offset += (data_size + 3) & ~3; @@ -456,7 +428,7 @@ int mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq, enum mail_cache_field field, const void *data, size_t data_size) { - uint32_t nb_data_size; + uint32_t data_size32; size_t full_size, offset; unsigned char *buf; int field_num; @@ -464,7 +436,7 @@ int mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq, i_assert(data_size > 0); i_assert(data_size < (uint32_t)-1); - nb_data_size = uint32_to_nbo((uint32_t)data_size); + data_size32 = (uint32_t)data_size; if ((field & MAIL_CACHE_FIXED_MASK) != 0) { field_num = get_field_num(field); @@ -484,7 +456,7 @@ int mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq, full_size = (data_size + 3) & ~3; if ((field & MAIL_CACHE_FIXED_MASK) == 0) - full_size += sizeof(nb_data_size); + full_size += sizeof(data_size32); /* fields must be ordered. find where to insert it. */ if (field > ctx->cache_rec.fields) @@ -500,8 +472,8 @@ int mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq, /* @UNSAFE */ if ((field & MAIL_CACHE_FIXED_MASK) == 0) { - memcpy(buf, &nb_data_size, sizeof(nb_data_size)); - buf += sizeof(nb_data_size); + memcpy(buf, &data_size32, sizeof(data_size32)); + buf += sizeof(data_size32); } memcpy(buf, data, data_size); buf += data_size; if ((data_size & 3) != 0) @@ -533,12 +505,12 @@ int mail_cache_delete(struct mail_cache_transaction_ctx *ctx, uint32_t seq) the data. also it's actually useful as some index views are still able to ask cached data from messages that have already been expunged. */ - deleted_space = nbo_to_uint32(cache->hdr->deleted_space); + deleted_space = cache->hdr->deleted_space; do { - deleted_space -= nbo_to_uint32(cache_rec->size); - cache_rec = mail_cache_get_record(cache, cache_rec->next_offset, - FALSE); + deleted_space += cache_rec->size; + cache_rec = + mail_cache_get_record(cache, cache_rec->prev_offset); } while (cache_rec != NULL); /* see if we've reached the max. deleted space in file */ @@ -547,7 +519,7 @@ int mail_cache_delete(struct mail_cache_transaction_ctx *ctx, uint32_t seq) ctx->used_file_size >= COMPRESS_MIN_SIZE) cache->need_compress = TRUE; - cache->hdr->deleted_space = uint32_to_nbo(deleted_space); + cache->hdr->deleted_space = deleted_space; return 0; } @@ -580,3 +552,22 @@ int mail_cache_update_record_flags(struct mail_cache_view *view, uint32_t seq, { return -1; } + +int mail_cache_link(struct mail_cache *cache, uint32_t old_offset, + uint32_t new_offset) +{ + struct mail_cache_record *cache_rec; + + i_assert(cache->locks > 0); + + if (mail_cache_map(cache, new_offset, sizeof(*cache_rec)) < 0) + return -1; + + if (new_offset + sizeof(*cache_rec) > cache->mmap_length) { + mail_cache_set_corrupted(cache, "record points outside file"); + return -1; + } + cache_rec = CACHE_RECORD(cache, new_offset); + cache_rec->prev_offset = old_offset; + return 0; +} diff --git a/src/lib-index/mail-cache.c b/src/lib-index/mail-cache.c index e04e2f7688..a0389c8970 100644 --- a/src/lib-index/mail-cache.c +++ b/src/lib-index/mail-cache.c @@ -1,7 +1,6 @@ /* Copyright (C) 2003-2004 Timo Sirainen */ #include "lib.h" -#include "byteorder.h" #include "file-lock.h" #include "file-set-size.h" #include "mmap-util.h" @@ -156,7 +155,6 @@ int mail_cache_reopen(struct mail_cache *cache) static int mmap_verify_header(struct mail_cache *cache) { struct mail_cache_header *hdr; - uint32_t used_file_size; /* check that the header is still ok */ if (cache->mmap_length < sizeof(struct mail_cache_header)) { @@ -165,6 +163,11 @@ static int mmap_verify_header(struct mail_cache *cache) } cache->hdr = hdr = cache->mmap_base; + if (cache->hdr->version != MAIL_CACHE_VERSION) { + /* version changed - upgrade silently */ + return FALSE; + } + if (cache->hdr->indexid != cache->index->indexid) { /* index id changed */ mail_cache_set_corrupted(cache, "indexid changed"); @@ -180,17 +183,16 @@ static int mmap_verify_header(struct mail_cache *cache) if (cache->locks == 0) return TRUE; - used_file_size = nbo_to_uint32(hdr->used_file_size); - if (used_file_size < sizeof(struct mail_cache_header)) { + if (hdr->used_file_size < sizeof(struct mail_cache_header)) { mail_cache_set_corrupted(cache, "used_file_size too small"); return FALSE; } - if ((used_file_size % sizeof(uint32_t)) != 0) { + if ((hdr->used_file_size % sizeof(uint32_t)) != 0) { mail_cache_set_corrupted(cache, "used_file_size not aligned"); return FALSE; } - if (used_file_size > cache->mmap_length) { + if (hdr->used_file_size > cache->mmap_length) { mail_cache_set_corrupted(cache, "used_file_size too large"); return FALSE; } @@ -354,17 +356,13 @@ int mail_cache_lock(struct mail_cache *cache, int nonblock) return ret; } -int mail_cache_unlock(struct mail_cache *cache) +void mail_cache_unlock(struct mail_cache *cache) { if (--cache->locks > 0) - return 0; + return; - if (file_wait_lock(cache->fd, F_UNLCK) <= 0) { + if (file_wait_lock(cache->fd, F_UNLCK) <= 0) mail_cache_set_syscall_error(cache, "file_wait_lock(F_UNLCK)"); - return -1; - } - - return 0; } int mail_cache_is_locked(struct mail_cache *cache) diff --git a/src/lib-index/mail-cache.h b/src/lib-index/mail-cache.h index 9b889de5a2..8b81ea3079 100644 --- a/src/lib-index/mail-cache.h +++ b/src/lib-index/mail-cache.h @@ -86,7 +86,7 @@ int mail_cache_compress(struct mail_cache *cache, struct mail_index_view *view); /* Explicitly lock the cache file. Returns -1 if error, 1 if ok, 0 if we couldn't lock */ int mail_cache_lock(struct mail_cache *cache, int nonblock); -int mail_cache_unlock(struct mail_cache *cache); +void mail_cache_unlock(struct mail_cache *cache); /* Returns TRUE if cache file is locked. */ int mail_cache_is_locked(struct mail_cache *cache); @@ -106,8 +106,7 @@ int mail_cache_transaction_begin(struct mail_cache_view *view, int nonblock, int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx); void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx); -/* Should be called only by mail_transaction_commit/rollback: */ -int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx); +void mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx); /* Return NULL-terminated list of headers for given index, or NULL if header index isn't used. */ diff --git a/src/lib-index/mail-index-sync-private.h b/src/lib-index/mail-index-sync-private.h index e3260c4a1f..1814a53ab0 100644 --- a/src/lib-index/mail-index-sync-private.h +++ b/src/lib-index/mail-index-sync-private.h @@ -22,6 +22,7 @@ struct mail_index_sync_ctx { unsigned int lock_id; unsigned int sync_appends:1; + unsigned int cache_locked:1; }; extern struct mail_transaction_map_functions mail_index_map_sync_funcs; diff --git a/src/lib-index/mail-index-sync-update.c b/src/lib-index/mail-index-sync-update.c index 3f9443c673..bf135d8a1a 100644 --- a/src/lib-index/mail-index-sync-update.c +++ b/src/lib-index/mail-index-sync-update.c @@ -9,6 +9,7 @@ #include "mail-index-sync-private.h" #include "mail-transaction-log.h" #include "mail-transaction-util.h" +#include "mail-cache-private.h" #include @@ -58,7 +59,8 @@ mail_index_header_update_lowwaters(struct mail_index_header *hdr, static int sync_expunge(const struct mail_transaction_expunge *e, void *context) { - struct mail_index_view *view = context; + struct mail_index_sync_ctx *sync_ctx = context; + struct mail_index_view *view = sync_ctx->view; struct mail_index_map *map = view->map; struct mail_index_header *hdr = &map->hdr_copy; struct mail_index_record *rec; @@ -98,7 +100,8 @@ static int sync_expunge(const struct mail_transaction_expunge *e, void *context) static int sync_append(const struct mail_transaction_append_header *hdr, const struct mail_index_record *rec, void *context) { - struct mail_index_view *view = context; + struct mail_index_sync_ctx *sync_ctx = context; + struct mail_index_view *view = sync_ctx->view; struct mail_index_map *map = view->map; void *dest; @@ -139,7 +142,8 @@ static int sync_append(const struct mail_transaction_append_header *hdr, static int sync_flag_update(const struct mail_transaction_flag_update *u, void *context) { - struct mail_index_view *view = context; + struct mail_index_sync_ctx *sync_ctx = context; + struct mail_index_view *view = sync_ctx->view; struct mail_index_record *rec; struct mail_index_header *hdr; uint8_t flag_mask, old_flags; @@ -188,7 +192,8 @@ static int sync_flag_update(const struct mail_transaction_flag_update *u, static int sync_cache_reset(const struct mail_transaction_cache_reset *u, void *context) { - struct mail_index_view *view = context; + struct mail_index_sync_ctx *sync_ctx = context; + struct mail_index_view *view = sync_ctx->view; uint32_t i; view->map->hdr_copy.cache_file_seq = u->new_file_seq; @@ -201,7 +206,9 @@ static int sync_cache_reset(const struct mail_transaction_cache_reset *u, static int sync_cache_update(const struct mail_transaction_cache_update *u, void *context) { - struct mail_index_view *view = context; + struct mail_index_sync_ctx *sync_ctx = context; + struct mail_index_view *view = sync_ctx->view; + struct mail_index_record *rec; uint32_t seq; int ret; @@ -209,10 +216,25 @@ static int sync_cache_update(const struct mail_transaction_cache_update *u, &seq, &seq); i_assert(ret == 0); - if (seq != 0) { - MAIL_INDEX_MAP_IDX(view->map, seq-1)->cache_offset = - u->cache_offset; + if (seq == 0) { + /* already expunged */ + return 1; } + + rec = MAIL_INDEX_MAP_IDX(view->map, seq-1); + if (rec->cache_offset != 0) { + /* we'll need to link the old and new cache records */ + if (!sync_ctx->cache_locked) { + if (mail_cache_lock(view->index->cache, FALSE) <= 0) + return -1; + sync_ctx->cache_locked = TRUE; + } + + if (mail_cache_link(view->index->cache, + rec->cache_offset, u->cache_offset) < 0) + return -1; + } + rec->cache_offset = u->cache_offset; return 1; } @@ -423,12 +445,17 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx) if (mail_transaction_map(index, hdr, data, &mail_index_map_sync_funcs, - view) < 0) { + sync_ctx) < 0) { ret = -1; break; } } + if (sync_ctx->cache_locked) { + mail_cache_unlock(index->cache); + sync_ctx->cache_locked = FALSE; + } + if (ret < 0) { mail_index_view_unlock(view); return -1; diff --git a/src/lib-storage/index/index-mail-headers.c b/src/lib-storage/index/index-mail-headers.c index 76a4dd17ca..83805a791b 100644 --- a/src/lib-storage/index/index-mail-headers.c +++ b/src/lib-storage/index/index-mail-headers.c @@ -792,7 +792,9 @@ void index_mail_headers_close(struct index_mail *mail) len = str_len(mail->data.header_data) - data->header_data_uncached_offset; - mail_cache_add(mail->trans->cache_trans, data->seq, - mail_cache_header_fields[idx], str, len+1); + if (len != 0) { + mail_cache_add(mail->trans->cache_trans, data->seq, + mail_cache_header_fields[idx], str, len+1); + } data->header_save = FALSE; } diff --git a/src/lib-storage/index/index-transaction.c b/src/lib-storage/index/index-transaction.c index 159be2b82e..616e19b860 100644 --- a/src/lib-storage/index/index-transaction.c +++ b/src/lib-storage/index/index-transaction.c @@ -16,7 +16,7 @@ void index_transaction_init(struct index_transaction_context *t, static void index_transaction_free(struct index_transaction_context *t) { if (t->cache_trans != NULL) - (void)mail_cache_transaction_end(t->cache_trans); + mail_cache_transaction_end(t->cache_trans); mail_cache_view_close(t->cache_view); mail_index_view_close(t->trans_view);