#include "byteorder.h"
#include "mail-cache-private.h"
+#define CACHE_PREFETCH 1024
+
const char *
mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx)
{
if (offset == 0)
return NULL;
- if (mail_cache_mmap_update(cache, offset, 1024) < 0)
+ if (mail_cache_mmap_update(cache, offset, CACHE_PREFETCH) < 0)
return NULL;
if (offset + sizeof(data_size) > cache->mmap_length) {
return NULL;
}
- if (mail_cache_mmap_update(cache, offset, data_size) < 0)
- return NULL;
+ if (data_size + sizeof(data_size) > CACHE_PREFETCH) {
+ if (mail_cache_mmap_update(cache, offset, data_size) < 0)
+ return NULL;
+ }
if (offset + data_size > cache->mmap_length) {
mail_cache_set_corrupted(cache, "Header %u points outside file",
mail_cache_get_record(struct mail_cache *cache, uint32_t offset,
int index_offset)
{
-#define CACHE_PREFETCH 1024
struct mail_cache_record *cache_rec;
size_t size;
enum mail_cache_field fields)
{
const struct mail_index_record *rec;
+ struct mail_index_map *map;
if (mail_cache_transaction_autocommit(view, seq, fields) < 0)
return NULL;
- /* FIXME: check cache_offset in transaction
- FIXME: if rec doesn't point to header record, the file seq may
- be different and the offset wrong */
- if (mail_index_lookup(view->view, seq, &rec) < 0)
+
+ if (view->cache->disabled)
+ return NULL;
+
+ /* FIXME: check cache_offset in transaction */
+ if (mail_index_lookup_full(view->view, seq, &map, &rec) < 0)
return NULL;
+ if (map->hdr->cache_file_seq != view->cache->hdr->file_seq) {
+ /* FIXME: we should check if newer file is available? */
+ return NULL;
+ }
+
return mail_cache_get_record(view->cache, rec->cache_offset, TRUE);
}
void *mmap_base;
size_t mmap_length;
- uint32_t used_file_size;
struct mail_cache_header *hdr;
unsigned int mmap_refresh:1;
unsigned int silent:1;
+ unsigned int disabled:1;
};
struct mail_cache_view {
uint32_t first_seq, last_seq, prev_seq;
enum mail_cache_field prev_fields;
buffer_t *cache_marks;
+ uint32_t used_file_size;
};
static const unsigned char *null4[] = { 0, 0, 0, 0 };
ctx->view = view;
ctx->trans = t;
ctx->cache_data = buffer_create_dynamic(system_pool, 8192, (size_t)-1);
+ ctx->used_file_size = nbo_to_uint32(ctx->cache->hdr->used_file_size);
view->cache->trans_ctx = ctx;
*ctx_r = ctx;
if (cont * 100 / cache->index->hdr->messages_count >=
COMPRESS_CONTINUED_PERCENTAGE &&
- cache->used_file_size >= COMPRESS_MIN_SIZE) {
+ ctx->used_file_size >= COMPRESS_MIN_SIZE) {
/* too many continued rows, compress */
//FIXME:cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
}
return 0;
}
-static int mail_cache_grow(struct mail_cache *cache, uint32_t size)
+static int
+mail_cache_grow(struct mail_cache_transaction_ctx *ctx, uint32_t size)
{
+ struct mail_cache *cache = ctx->cache;
struct stat st;
uoff_t grow_size, new_fsize;
- new_fsize = cache->used_file_size + size;
+ new_fsize = ctx->used_file_size + size;
grow_size = new_fsize / 100 * MAIL_CACHE_GROW_PERCENTAGE;
if (grow_size < 16384)
grow_size = 16384;
return -1;
}
- if (cache->used_file_size + size <= (uoff_t)st.st_size) {
+ if (ctx->used_file_size + size <= (uoff_t)st.st_size) {
/* no need to grow, just update mmap */
if (mail_cache_mmap_update(cache, 0, 0) < 0)
return -1;
i_assert((size & 3) == 0);
- offset = ctx->cache->used_file_size;
+ offset = ctx->used_file_size;
if (offset >= 0x40000000) {
mail_index_set_error(ctx->cache->index,
"Cache file too large: %s",
}
if (offset + size > ctx->cache->mmap_length) {
- if (mail_cache_grow(ctx->cache, size) < 0)
+ if (mail_cache_grow(ctx, size) < 0)
return 0;
}
- ctx->cache->used_file_size += size;
+ ctx->used_file_size += size;
return offset;
}
struct mail_cache *cache = ctx->cache;
struct mail_cache_record *cache_rec, *next;
const struct mail_index_record *rec;
+ struct mail_index_map *map;
uint32_t write_offset, update_offset;
const void *buf;
size_t size, buf_size;
ctx->cache_rec.size = uint32_to_nbo(size);
// FIXME: check cache_offset in transaction
- ret = mail_index_lookup(ctx->view->view, ctx->prev_seq, &rec);
+ ret = mail_index_lookup_full(ctx->view->view, ctx->prev_seq,
+ &map, &rec);
if (ret < 0)
return -1;
+ if (map->hdr->cache_file_seq != cache->hdr->file_seq) {
+ /* FIXME: we should check if newer file is available? */
+ ret = 0;
+ }
+
if (ret == 0) {
/* it's been expunged already, do nothing */
} else {
{
int ret = 0;
+ if (ctx->cache->disabled) {
+ mail_cache_transaction_flush(ctx);
+ return 0;
+ }
+
if (ctx->prev_seq != 0) {
if (mail_cache_write(ctx) < 0)
return -1;
}
- ctx->cache->hdr->used_file_size =
- uint32_to_nbo(ctx->cache->used_file_size);
+ ctx->cache->hdr->used_file_size = uint32_to_nbo(ctx->used_file_size);
if (commit_all_changes(ctx) < 0)
ret = -1;
/* no need to actually modify the file - we just didn't update
used_file_size */
- cache->used_file_size = nbo_to_uint32(cache->hdr->used_file_size);
+ ctx->used_file_size = nbo_to_uint32(cache->hdr->used_file_size);
/* make sure we don't cache the headers */
for (i = 0; i < ctx->next_unused_header_lowwater; i++) {
} while (cache_rec != NULL);
/* see if we've reached the max. deleted space in file */
- max_del_space = cache->used_file_size / 100 * COMPRESS_PERCENTAGE;
+ max_del_space = ctx->used_file_size / 100 * COMPRESS_PERCENTAGE;
if (deleted_space >= max_del_space &&
- cache->used_file_size >= COMPRESS_MIN_SIZE) {
+ ctx->used_file_size >= COMPRESS_MIN_SIZE) {
//FIXME:cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
}
static int mmap_verify_header(struct mail_cache *cache)
{
struct mail_cache_header *hdr;
+ uint32_t used_file_size;
/* check that the header is still ok */
if (cache->mmap_length < sizeof(struct mail_cache_header)) {
return 1;
}
- cache->used_file_size = nbo_to_uint32(hdr->used_file_size);
-
/* only check the header if we're locked */
if (cache->locks == 0)
return 1;
- if (cache->used_file_size < sizeof(struct mail_cache_header)) {
+ used_file_size = nbo_to_uint32(hdr->used_file_size);
+ if (used_file_size < sizeof(struct mail_cache_header)) {
mail_cache_set_corrupted(cache, "used_file_size too small");
return 0;
}
- if ((cache->used_file_size % sizeof(uint32_t)) != 0) {
+ if ((used_file_size % sizeof(uint32_t)) != 0) {
mail_cache_set_corrupted(cache, "used_file_size not aligned");
return 0;
}
- if (cache->used_file_size > cache->mmap_length) {
- /* maybe a crash truncated the file - just fix it */
- hdr->used_file_size = uint32_to_nbo(cache->mmap_length & ~3);
- if (msync(cache->mmap_base, sizeof(*hdr), MS_SYNC) < 0) {
- mail_cache_set_syscall_error(cache, "msync()");
- return -1;
- }
+ if (used_file_size > cache->mmap_length) {
+ mail_cache_set_corrupted(cache, "used_file_size too large");
+ return 0;
}
return 1;
}
return -1;
}
- if (cache->index->hdr->cache_file_seq != 0) {
- // FIXME: clear cache_offsets in index file
- }
-
mail_cache_file_close(cache);
cache->fd = dup(fd);
cache->fd = -1;
cache->split_header_pool = pool_alloconly_create("Headers", 512);
- /* we'll do anon-mmaping only if initially requested. if we fail
- because of out of disk space, we'll just let the main index code
- know it and fail. */
if (mail_cache_open_or_create_file(cache, &hdr) < 0) {
- mail_cache_free(cache);
- return NULL;
+ /* failed for some reason - doesn't really matter,
+ just disable caching. */
+ mail_cache_file_close(cache);
+
+ i_free(cache->filepath);
+ cache->filepath = i_strdup_printf("(disabled cache for %s)",
+ index->filepath);
+ cache->disabled = TRUE;
}
return cache;
struct mail_cache_header hdr;
int fd;
- i_assert(cache->index->lock_type == F_WRLCK);
-
memset(&hdr, 0, sizeof(hdr));
hdr.indexid = cache->index->indexid;
hdr.file_seq = cache->index->hdr->cache_file_seq + 1;
hdr.used_file_size = uint32_to_nbo(sizeof(hdr));
- cache->used_file_size = sizeof(hdr);
-
- // FIXME: update cache_offsets in index
fd = file_dotlock_open(cache->filepath, NULL, NULL,
MAIL_CACHE_LOCK_TIMEOUT,
(void)mail_cache_unlock(cache);
return -1;
}
-#if 0 // FIXME
+
if (cache->hdr->file_seq != cache->index->hdr->cache_file_seq) {
- /* we have the cache file locked and sync_id still
- doesn't match. it means we crashed between updating
- cache file and updating sync_id in index header.
- just update the sync_ids so they match. */
- i_warning("Updating broken sync_id in cache file %s",
- cache->filepath);
- cache->hdr->file_seq =
- cache->index->hdr->cache_file_seq;
+ mail_cache_unlock(cache);
+ return 0;
}
-#endif
}
return ret;
}
return cache->locks > 0;
}
+int mail_cache_need_reset(struct mail_cache *cache, uint32_t *new_file_seq_r)
+{
+ if (cache->hdr->file_seq != cache->index->hdr->cache_file_seq) {
+ if (mail_cache_lock(cache, TRUE) == 0) {
+ *new_file_seq_r = cache->index->hdr->cache_file_seq;
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
struct mail_cache_view *
mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *iview)
{
/* Reset the cache file, clearing all data. */
int mail_cache_reset(struct mail_cache *cache);
-/* Explicitly lock the cache file. Returns 1 if ok, 0 if nonblock is TRUE and
- we couldn't immediately get a lock, or -1 if error. */
+/* Explicitly lock the cache file. Returns -1 if error, 1 if ok,
+ 0 if we couldn't lock (nonblock = TRUE or index file needs a cache reset) */
int mail_cache_lock(struct mail_cache *cache, int nonblock);
int mail_cache_unlock(struct mail_cache *cache);
/* Returns TRUE if cache file is locked. */
int mail_cache_is_locked(struct mail_cache *cache);
+/* Returns TRUE if index's cache_file_seq doesn't match the latest cache file */
+int mail_cache_need_reset(struct mail_cache *cache, uint32_t *new_file_seq_r);
+
struct mail_cache_view *
mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *iview);
void mail_cache_view_close(struct mail_cache_view *view);
struct mail_index_map *
mail_index_map_to_memory(struct mail_index *index, struct mail_index_map *map);
+int mail_index_lookup_full(struct mail_index_view *view, uint32_t seq,
+ struct mail_index_map **map_r,
+ const struct mail_index_record **rec_r);
+
+void mail_index_reset_cache(struct mail_index_transaction *t,
+ uint32_t new_file_seq);
void mail_index_update_cache(struct mail_index_transaction *t,
uint32_t seq, uint32_t offset);
return 1;
}
+static int sync_cache_reset(const struct mail_transaction_cache_reset *u,
+ void *context)
+{
+ struct mail_index_view *view = context;
+ uint32_t i;
+
+ view->map->hdr_copy.cache_file_seq = u->new_file_seq;
+
+ for (i = 0; i < view->messages_count; i++)
+ MAIL_INDEX_MAP_IDX(view->index, view->map, i)->cache_offset = 0;
+ return 1;
+}
+
static int sync_cache_update(const struct mail_transaction_cache_update *u,
void *context)
{
struct mail_transaction_map_functions mail_index_map_sync_funcs = {
sync_expunge, sync_append, sync_flag_update,
- sync_cache_update, sync_header_update, sync_extra_rec_update
+ sync_cache_reset, sync_cache_update, sync_header_update,
+ sync_extra_rec_update
};
uint32_t log_file_seq, uoff_t log_file_offset)
{
struct mail_index_sync_ctx *ctx;
- uint32_t seq;
+ uint32_t seq, new_file_seq;
uoff_t offset;
unsigned int lock_id;
return -1;
}
+ /* check here if cache file's sequence has changed unexpectedly */
+ if (mail_cache_need_reset(index->cache, &new_file_seq)) {
+ uint32_t seq;
+ uoff_t offset;
+ struct mail_index_transaction *t;
+
+ t = mail_index_transaction_begin(ctx->view, FALSE);
+ mail_index_reset_cache(t, new_file_seq);
+ mail_index_transaction_commit(t, &seq, &offset);
+ }
+
*ctx_r = ctx;
*view_r = ctx->view;
return 1;
unsigned char hdr_change[sizeof(struct mail_index_header)];
unsigned char hdr_mask[sizeof(struct mail_index_header)];
+ uint32_t new_cache_file_seq;
buffer_t *extra_rec_updates[MAIL_INDEX_MAX_EXTRA_RECORDS];
memcpy(seq_p+1, record, record_size);
}
+void mail_index_reset_cache(struct mail_index_transaction *t,
+ uint32_t new_file_seq)
+{
+ t->new_cache_file_seq = new_file_seq;
+}
+
void mail_index_update_cache(struct mail_index_transaction *t,
uint32_t seq, uint32_t offset)
{
return 0;
}
-static int mail_index_lookup_int(struct mail_index_view *view, uint32_t seq,
- struct mail_index_map **map_r,
- const struct mail_index_record **rec_r)
+int mail_index_lookup_full(struct mail_index_view *view, uint32_t seq,
+ struct mail_index_map **map_r,
+ const struct mail_index_record **rec_r)
{
struct mail_index_map *map;
const struct mail_index_record *rec, *n_rec;
{
struct mail_index_map *map;
- return mail_index_lookup_int(view, seq, &map, rec_r);
+ return mail_index_lookup_full(view, seq, &map, rec_r);
}
int mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq,
uint32_t offset;
int ret;
- if ((ret = mail_index_lookup_int(view, seq, &map, &rec)) < 0)
+ if ((ret = mail_index_lookup_full(view, seq, &map, &rec)) < 0)
return -1;
if (rec == NULL) {
return 0;
}
+static const buffer_t *get_cache_reset_buf(struct mail_index_transaction *t)
+{
+ struct mail_transaction_cache_reset u;
+ buffer_t *buf;
+
+ memset(&u, 0, sizeof(u));
+ u.new_file_seq = t->new_cache_file_seq;
+
+ buf = buffer_create_static(pool_datastack_create(), sizeof(u));
+ buffer_append(buf, &u, sizeof(u));
+ return buf;
+}
+
static const buffer_t *
log_get_hdr_update_buffer(struct mail_index_transaction *t)
{
MAIL_TRANSACTION_FLAG_UPDATE,
view->external);
}
+ if (t->new_cache_file_seq != 0) {
+ ret = log_append_buffer(file, get_cache_reset_buf(t), NULL,
+ MAIL_TRANSACTION_CACHE_RESET,
+ view->external);
+ }
if (t->cache_updates != NULL && ret == 0) {
ret = log_append_buffer(file, t->cache_updates, NULL,
MAIL_TRANSACTION_CACHE_UPDATE,
MAIL_TRANSACTION_EXPUNGE = 0x00000001,
MAIL_TRANSACTION_APPEND = 0x00000002,
MAIL_TRANSACTION_FLAG_UPDATE = 0x00000004,
- MAIL_TRANSACTION_CACHE_UPDATE = 0x00000008,
- MAIL_TRANSACTION_HEADER_UPDATE = 0x00000010,
- MAIL_TRANSACTION_EXTRA_REC_UPDATE = 0x00000020,
+ MAIL_TRANSACTION_CACHE_RESET = 0x00000008,
+ MAIL_TRANSACTION_CACHE_UPDATE = 0x00000010,
+ MAIL_TRANSACTION_HEADER_UPDATE = 0x00000020,
+ MAIL_TRANSACTION_EXTRA_REC_UPDATE = 0x00000040,
MAIL_TRANSACTION_TYPE_MASK = 0x0000ffff,
keywords_mask_t remove_keywords;
};
+struct mail_transaction_cache_reset {
+ uint32_t new_file_seq;
+};
+
struct mail_transaction_cache_update {
uint32_t uid;
uint32_t cache_offset;
sizeof(struct mail_transaction_expunge) },
{ MAIL_TRANSACTION_FLAG_UPDATE, MAIL_INDEX_SYNC_TYPE_FLAGS,
sizeof(struct mail_transaction_flag_update) },
+ { MAIL_TRANSACTION_CACHE_RESET, 0,
+ sizeof(struct mail_transaction_cache_reset) },
{ MAIL_TRANSACTION_CACHE_UPDATE, 0,
sizeof(struct mail_transaction_cache_update) },
{ MAIL_TRANSACTION_HEADER_UPDATE, 0, 1 }, /* variable size, use 1 */
}
break;
}
+ case MAIL_TRANSACTION_CACHE_RESET: {
+ const struct mail_transaction_cache_reset *u = data;
+
+ if (map->cache_reset != NULL)
+ ret = map->cache_reset(u, context);
+ break;
+ }
case MAIL_TRANSACTION_CACHE_UPDATE: {
const struct mail_transaction_cache_update *rec, *end;
int (*append)(const struct mail_index_record *rec, void *context);
int (*flag_update)(const struct mail_transaction_flag_update *u,
void *context);
+ int (*cache_reset)(const struct mail_transaction_cache_reset *u,
+ void *context);
int (*cache_update)(const struct mail_transaction_cache_update *u,
void *context);
int (*header_update)(const struct mail_transaction_header_update *u,
projects / thirdparty / dovecot / core.git / commitdiff
