+++ /dev/null
-/* Copyright (C) 2003 Timo Sirainen */
-
-#include "lib.h"
-#include "buffer.h"
-#include "byteorder.h"
-#include "file-lock.h"
-#include "file-set-size.h"
-#include "ioloop.h"
-#include "mmap-util.h"
-#include "write-full.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-
-#include <stddef.h>
-#include <unistd.h>
-#include <sys/stat.h>
-
-/* Never compress the file if it's smaller than this */
-#define COMPRESS_MIN_SIZE (1024*50)
-
-/* Compress the file when deleted space reaches n% of total size */
-#define COMPRESS_PERCENTAGE 20
-
-/* Compress the file when n% of rows contain continued rows.
- 200% means that there's 2 continued rows per record. */
-#define COMPRESS_CONTINUED_PERCENTAGE 200
-
-/* Initial size for the file */
-#define MAIL_CACHE_INITIAL_SIZE (sizeof(struct mail_cache_header) + 10240)
-
-/* When more space is needed, grow the file n% larger than the previous size */
-#define MAIL_CACHE_GROW_PERCENTAGE 10
-
-#define MAIL_CACHE_LOCK_TIMEOUT 120
-#define MAIL_CACHE_LOCK_CHANGE_TIMEOUT 60
-#define MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT (5*60)
-
-#define CACHE_RECORD(cache, offset) \
- ((struct mail_cache_record *) ((char *) (cache)->mmap_base + offset))
-
-struct mail_cache_header {
- uint32_t indexid;
- uint32_t sync_id;
-
- uint32_t continued_record_count;
-
- uint32_t used_file_size;
- uint32_t deleted_space;
-
- uint32_t used_fields; /* enum mail_cache_field */
-
- uint32_t field_usage_start; /* time_t */
- uint32_t field_usage_counts[32];
-
- uint32_t header_offsets[MAIL_CACHE_HEADERS_COUNT];
-};
-
-struct mail_cache_record {
- uint32_t fields; /* enum mail_cache_field */
- uint32_t next_offset;
- uint32_t size; /* full record size, including this header */
-};
-
-struct mail_cache {
- struct mail_index *index;
-
- char *filepath;
- int fd;
-
- void *mmap_base;
- size_t mmap_length;
- uint32_t used_file_size;
- uint32_t sync_id;
-
- struct mail_cache_header *header;
-
- pool_t split_header_pool;
- uint32_t split_offsets[MAIL_CACHE_HEADERS_COUNT];
- const char *const *split_headers[MAIL_CACHE_HEADERS_COUNT];
-
- enum mail_cache_field default_cache_fields;
- enum mail_cache_field never_cache_fields;
-
- struct mail_cache_transaction_ctx *trans_ctx;
- unsigned int locks;
-
- unsigned int anon_mmap:1;
- unsigned int mmap_refresh:1;
- unsigned int silent:1;
-};
-
-struct mail_cache_transaction_ctx {
- struct mail_cache *cache;
-
- unsigned int next_unused_header_lowwater;
-
- unsigned int last_idx;
- struct mail_cache_record cache_rec;
- buffer_t *cache_data;
-
- unsigned int first_uid, last_uid, prev_uid;
- enum mail_cache_field prev_fields;
- buffer_t *index_marks, *cache_marks;
-};
-
-unsigned int mail_cache_field_sizes[32] = {
- sizeof(enum mail_index_record_flag),
- sizeof(uoff_t),
- 16,
- sizeof(struct mail_sent_date),
- sizeof(time_t),
- sizeof(uoff_t),
- sizeof(uoff_t),
-
- 0, 0, 0, 0, 0,
-
- /* variable sized */
- (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
- (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
- (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
- (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
- (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1
-};
-
-enum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT] = {
- MAIL_CACHE_HEADERS1,
- MAIL_CACHE_HEADERS2,
- MAIL_CACHE_HEADERS3,
- MAIL_CACHE_HEADERS4
-};
-
-static const unsigned char *null4[] = { 0, 0, 0, 0 };
-
-static const char *
-mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx);
-static int mail_cache_write(struct mail_cache_transaction_ctx *ctx);
-static struct mail_cache_record *
-mail_cache_lookup(struct mail_cache *cache,
- const struct mail_index_record *rec,
- enum mail_cache_field fields);
-
-static uint32_t uint32_to_offset(uint32_t offset)
-{
- unsigned char buf[4];
-
- i_assert(offset < 0x40000000);
- i_assert((offset & 3) == 0);
-
- offset >>= 2;
- buf[0] = 0x80 | ((offset & 0x0fe00000) >> 21);
- buf[1] = 0x80 | ((offset & 0x001fc000) >> 14);
- buf[2] = 0x80 | ((offset & 0x00003f80) >> 7);
- buf[3] = 0x80 | (offset & 0x0000007f);
- return *((uint32_t *) buf);
-}
-
-static uint32_t offset_to_uint32(uint32_t offset)
-{
- const unsigned char *buf = (const unsigned char *) &offset;
-
- if ((offset & 0x80808080) != 0x80808080)
- return 0;
-
- return (((uint32_t)buf[3] & 0x7f) << 2) |
- (((uint32_t)buf[2] & 0x7f) << 9) |
- (((uint32_t)buf[1] & 0x7f) << 16) |
- (((uint32_t)buf[0] & 0x7f) << 23);
-}
-
-static int mail_cache_set_syscall_error(struct mail_cache *cache,
- const char *function)
-{
- i_assert(function != NULL);
-
- if (ENOSPACE(errno)) {
- cache->index->nodiskspace = TRUE;
- return FALSE;
- }
-
- index_set_error(cache->index, "%s failed with index cache file %s: %m",
- function, cache->filepath);
- return FALSE;
-}
-
-static int mail_cache_create_memory(struct mail_cache *cache,
- struct mail_cache_header *hdr)
-{
- cache->mmap_length = MAIL_CACHE_INITIAL_SIZE;
- cache->mmap_base = mmap_anon(cache->mmap_length);
- if (cache->mmap_base == MAP_FAILED) {
- index_set_error(cache->index, "mmap_anon(%"PRIuSIZE_T")",
- cache->mmap_length);
- return FALSE;
- }
-
- cache->header = cache->mmap_base;
- *cache->header = *hdr;
-
- cache->anon_mmap = TRUE;
- cache->filepath = i_strdup_printf("(in-memory index cache for %s)",
- cache->index->mailbox_path);
- return TRUE;
-}
-
-static void mail_cache_file_close(struct mail_cache *cache)
-{
- if (cache->anon_mmap) {
- if (munmap_anon(cache->mmap_base, cache->mmap_length) < 0)
- mail_cache_set_syscall_error(cache, "munmap_anon()");
- } else if (cache->mmap_base != NULL) {
- if (munmap(cache->mmap_base, cache->mmap_length) < 0)
- mail_cache_set_syscall_error(cache, "munmap()");
- }
-
- cache->mmap_base = NULL;
- cache->header = NULL;
- cache->mmap_length = 0;
-
- if (cache->fd != -1) {
- if (close(cache->fd) < 0)
- mail_cache_set_syscall_error(cache, "close()");
- cache->fd = -1;
- }
-}
-
-static int mail_cache_file_reopen(struct mail_cache *cache)
-{
- int fd;
-
- if (cache->anon_mmap) {
- /* cache was set corrupted, we'll have to quit */
- return FALSE;
- }
-
- fd = open(cache->filepath, O_RDWR);
- if (fd == -1)
- return mail_cache_set_syscall_error(cache, "open()");
-
- mail_cache_file_close(cache);
-
- cache->fd = fd;
- return TRUE;
-}
-
-static int mmap_verify_header(struct mail_cache *cache)
-{
- struct mail_cache_header *hdr;
-
- /* check that the header is still ok */
- if (cache->mmap_length < sizeof(struct mail_cache_header))
- return mail_cache_set_corrupted(cache, "File too small");
- cache->header = hdr = cache->mmap_base;
- cache->sync_id = hdr->sync_id;
-
- if (cache->header->indexid != cache->index->indexid) {
- /* index id changed */
- if (cache->header->indexid != 0)
- mail_cache_set_corrupted(cache, "indexid changed");
- cache->index->inconsistent = TRUE; /* easiest way to rebuild */
- return FALSE;
- }
-
- if (cache->trans_ctx != NULL) {
- /* we've updated used_file_size, do nothing */
- return TRUE;
- }
-
- cache->used_file_size = nbo_to_uint32(hdr->used_file_size);
-
- /* only check the header if we're locked */
- if (cache->locks == 0)
- return TRUE;
-
- if (cache->used_file_size < sizeof(struct mail_cache_header)) {
- mail_cache_set_corrupted(cache, "used_file_size too small");
- return FALSE;
- }
- if ((cache->used_file_size % sizeof(uint32_t)) != 0) {
- mail_cache_set_corrupted(cache, "used_file_size not aligned");
- return FALSE;
- }
-
- if (cache->used_file_size > cache->mmap_length) {
- /* maybe a crash truncated the file - just fix it */
- hdr->used_file_size = uint32_to_nbo(cache->mmap_length & ~3);
- if (msync(cache->mmap_base, sizeof(*hdr), MS_SYNC) < 0)
- return mail_cache_set_syscall_error(cache, "msync()");
- }
- return TRUE;
-}
-
-static int mmap_update_nocheck(struct mail_cache *cache,
- size_t offset, size_t size)
-{
- struct stat st;
-
- /* if sync id has changed, the file has to be reopened.
- note that if main index isn't locked, it may change again */
- if (cache->sync_id != cache->index->cache_sync_id &&
- cache->mmap_base != NULL) {
- if (!mail_cache_file_reopen(cache))
- return -1;
- }
-
- if (offset < cache->mmap_length &&
- size <= cache->mmap_length - offset &&
- !cache->mmap_refresh) {
- /* already mapped */
- if (size != 0 || cache->anon_mmap)
- return 1;
-
- /* requesting the whole file - see if we need to
- re-mmap */
- if (fstat(cache->fd, &st) < 0) {
- mail_cache_set_syscall_error(cache, "fstat()");
- return -1;
- }
- if ((uoff_t)st.st_size == cache->mmap_length)
- return 1;
- }
- cache->mmap_refresh = FALSE;
-
- if (cache->anon_mmap)
- return 1;
-
- if (cache->mmap_base != NULL) {
- if (cache->locks != 0) {
- /* in the middle of transaction - write the changes */
- if (msync(cache->mmap_base, cache->mmap_length,
- MS_SYNC) < 0) {
- mail_cache_set_syscall_error(cache, "msync()");
- return -1;
- }
- }
-
- if (munmap(cache->mmap_base, cache->mmap_length) < 0)
- mail_cache_set_syscall_error(cache, "munmap()");
- }
-
- i_assert(cache->fd != -1);
-
- /* map the whole file */
- cache->header = NULL;
- cache->mmap_length = 0;
-
- cache->mmap_base = mmap_rw_file(cache->fd, &cache->mmap_length);
- if (cache->mmap_base == MAP_FAILED) {
- cache->mmap_base = NULL;
- mail_cache_set_syscall_error(cache, "mmap()");
- return -1;
- }
-
- /* re-mmaped, check header */
- return 0;
-}
-
-static int mmap_update(struct mail_cache *cache, size_t offset, size_t size)
-{
- int synced, ret;
-
- for (synced = FALSE;; synced = TRUE) {
- ret = mmap_update_nocheck(cache, offset, size);
- if (ret > 0)
- return TRUE;
- if (ret < 0)
- return FALSE;
-
- if (!mmap_verify_header(cache))
- return FALSE;
-
- /* see if cache file was rebuilt - do it only once to avoid
- infinite looping */
- if (cache->header->sync_id == cache->index->cache_sync_id ||
- synced)
- break;
-
- if (!mail_cache_file_reopen(cache))
- return FALSE;
- }
- return TRUE;
-}
-
-static int mail_cache_open_and_verify(struct mail_cache *cache, int silent)
-{
- struct stat st;
-
- mail_cache_file_close(cache);
-
- cache->fd = open(cache->filepath, O_RDWR);
- if (cache->fd == -1) {
- if (errno == ENOENT)
- return 0;
-
- mail_cache_set_syscall_error(cache, "open()");
- return -1;
- }
-
- if (fstat(cache->fd, &st) < 0) {
- mail_cache_set_syscall_error(cache, "fstat()");
- return -1;
- }
-
- if (st.st_size < sizeof(struct mail_cache_header))
- return 0;
-
- cache->mmap_refresh = TRUE;
- if (mmap_update_nocheck(cache, 0, sizeof(struct mail_cache_header)) < 0)
- return -1;
-
- /* verify that this really is the cache for wanted index */
- cache->silent = silent;
- if (!mmap_verify_header(cache)) {
- cache->silent = FALSE;
- return 0;
- }
-
- cache->silent = FALSE;
- return 1;
-}
-
-static void mail_index_clear_cache_offsets(struct mail_index *index)
-{
- struct mail_index_record *rec;
-
- index->sync_stamp = 0;
-
- rec = index->lookup(index, 1);
- while (rec != NULL) {
- rec->cache_offset = 0;
- rec = index->next(index, rec);
- }
-}
-
-static int mail_cache_open_or_create_file(struct mail_cache *cache,
- struct mail_cache_header *hdr)
-{
- int ret, fd;
-
- cache->filepath = i_strconcat(cache->index->filepath,
- MAIL_CACHE_FILE_PREFIX, NULL);
-
- ret = mail_cache_open_and_verify(cache, FALSE);
- if (ret != 0)
- return ret > 0;
-
- /* we'll have to clear cache_offsets which requires exclusive lock */
- cache->index->inconsistent = FALSE;
- if (!mail_index_set_lock(cache->index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
-
- /* maybe a rebuild.. */
- fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
- MAIL_CACHE_LOCK_CHANGE_TIMEOUT,
- MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL);
- if (fd == -1) {
- mail_cache_set_syscall_error(cache, "file_dotlock_open()");
- return FALSE;
- }
-
- /* see if someone else just created the cache file */
- ret = mail_cache_open_and_verify(cache, TRUE);
- if (ret != 0) {
- (void)file_dotlock_delete(cache->filepath, fd);
- return ret > 0;
- }
-
- /* rebuild then */
- if (write_full(fd, hdr, sizeof(*hdr)) < 0) {
- mail_cache_set_syscall_error(cache, "write_full()");
- (void)file_dotlock_delete(cache->filepath, fd);
- return FALSE;
- }
- if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) {
- mail_cache_set_syscall_error(cache, "file_set_size()");
- (void)file_dotlock_delete(cache->filepath, fd);
- return FALSE;
- }
-
- mail_index_clear_cache_offsets(cache->index);
-
- mail_cache_file_close(cache);
- cache->fd = dup(fd);
-
- if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
- mail_cache_set_syscall_error(cache, "file_dotlock_replace()");
- return FALSE;
- }
-
- cache->mmap_refresh = TRUE;
- if (!mmap_update(cache, 0, sizeof(struct mail_cache_header)))
- return FALSE;
-
- return TRUE;
-}
-
-int mail_cache_open_or_create(struct mail_index *index)
-{
- struct mail_cache_header hdr;
- struct mail_cache *cache;
-
- memset(&hdr, 0, sizeof(hdr));
- hdr.indexid = index->indexid;
- hdr.sync_id = index->cache_sync_id;
- hdr.used_file_size = uint32_to_nbo(sizeof(hdr));
-
- cache = i_new(struct mail_cache, 1);
- cache->index = index;
- cache->fd = -1;
- cache->split_header_pool = pool_alloconly_create("Headers", 512);
-
- index->cache = cache;
-
- /* we'll do anon-mmaping only if initially requested. if we fail
- because of out of disk space, we'll just let the main index code
- know it and fail. */
- if (INDEX_IS_IN_MEMORY(index)) {
- if (!mail_cache_create_memory(cache, &hdr)) {
- mail_cache_free(cache);
- return FALSE;
- }
- } else {
- if (!mail_cache_open_or_create_file(cache, &hdr)) {
- mail_cache_free(cache);
- return FALSE;
- }
- }
-
- /* unset inconsistency - we already rebuilt the cache file */
- index->inconsistent = FALSE;
-
- return TRUE;
-}
-
-void mail_cache_free(struct mail_cache *cache)
-{
- i_assert(cache->trans_ctx == NULL);
-
- cache->index->cache = NULL;
-
- mail_cache_file_close(cache);
-
- pool_unref(cache->split_header_pool);
- i_free(cache->filepath);
- i_free(cache);
-}
-
-void mail_cache_set_defaults(struct mail_cache *cache,
- enum mail_cache_field default_cache_fields,
- enum mail_cache_field never_cache_fields)
-{
- cache->default_cache_fields = default_cache_fields;
- cache->never_cache_fields = never_cache_fields;
-}
-
-static const struct mail_cache_record *
-mail_cache_compress_record(struct mail_cache *cache,
- struct mail_index_record *rec, int header_idx,
- uint32_t *size_r)
-{
- enum mail_cache_field orig_cached_fields, cached_fields, field;
- struct mail_cache_record cache_rec;
- buffer_t *buffer;
- const void *data;
- size_t size, pos;
- uint32_t nb_size;
- int i;
-
- memset(&cache_rec, 0, sizeof(cache_rec));
- buffer = buffer_create_dynamic(pool_datastack_create(),
- 4096, (size_t)-1);
-
- orig_cached_fields = mail_cache_get_fields(cache, rec);
- cached_fields = orig_cached_fields & ~MAIL_CACHE_HEADERS_MASK;
- buffer_append(buffer, &cache_rec, sizeof(cache_rec));
- for (i = 0, field = 1; i < 31; i++, field <<= 1) {
- if ((cached_fields & field) == 0)
- continue;
-
- if (!mail_cache_lookup_field(cache, rec, field, &data, &size)) {
- cached_fields &= ~field;
- continue;
- }
-
- nb_size = uint32_to_nbo((uint32_t)size);
-
- if ((field & MAIL_CACHE_FIXED_MASK) == 0)
- buffer_append(buffer, &nb_size, sizeof(nb_size));
- buffer_append(buffer, data, size);
- if ((size & 3) != 0)
- buffer_append(buffer, null4, 4 - (size & 3));
- }
-
- /* now merge all the headers if we have them all */
- if ((orig_cached_fields & mail_cache_header_fields[header_idx]) != 0) {
- nb_size = 0;
- pos = buffer_get_used_size(buffer);
- buffer_append(buffer, &nb_size, sizeof(nb_size));
-
- for (i = 0; i <= header_idx; i++) {
- field = mail_cache_header_fields[i];
- if (mail_cache_lookup_field(cache, rec, field,
- &data, &size) && size > 1) {
- size--; /* terminating \0 */
- buffer_append(buffer, data, size);
- nb_size += size;
- }
- }
- buffer_append(buffer, "", 1);
- nb_size++;
- if ((nb_size & 3) != 0)
- buffer_append(buffer, null4, 4 - (nb_size & 3));
-
- nb_size = uint32_to_nbo(nb_size);
- buffer_write(buffer, pos, &nb_size, sizeof(nb_size));
-
- cached_fields |= MAIL_CACHE_HEADERS1;
- }
-
- cache_rec.fields = cached_fields;
- cache_rec.size = uint32_to_nbo(buffer_get_used_size(buffer));
- buffer_write(buffer, 0, &cache_rec, sizeof(cache_rec));
-
- data = buffer_get_data(buffer, &size);
- *size_r = size;
- return data;
-}
-
-static int mail_cache_copy(struct mail_cache *cache, int fd)
-{
- struct mail_cache_header *hdr;
- const struct mail_cache_record *cache_rec;
- struct mail_index_record *rec;
- enum mail_cache_field used_fields;
- unsigned char *mmap_base;
- const char *str;
- uint32_t new_file_size, offset, size, nb_size;
- int i, header_idx;
-
- /* pick some reasonably good file size */
- new_file_size = cache->used_file_size -
- nbo_to_uint32(cache->header->deleted_space);
- new_file_size = (new_file_size + 1023) & ~1023;
- if (new_file_size < MAIL_CACHE_INITIAL_SIZE)
- new_file_size = MAIL_CACHE_INITIAL_SIZE;
-
- if (file_set_size(fd, new_file_size) < 0)
- return mail_cache_set_syscall_error(cache, "file_set_size()");
-
- mmap_base = mmap(NULL, new_file_size, PROT_READ | PROT_WRITE,
- MAP_SHARED, fd, 0);
- if (mmap_base == MAP_FAILED)
- return mail_cache_set_syscall_error(cache, "mmap()");
-
- /* skip file's header */
- hdr = (struct mail_cache_header *) mmap_base;
- offset = sizeof(*hdr);
-
- /* merge all the header pieces into one. if some message doesn't have
- all the required pieces, we'll just have to drop them all. */
- for (i = MAIL_CACHE_HEADERS_COUNT-1; i >= 0; i--) {
- str = mail_cache_get_header_fields_str(cache, i);
- if (str != NULL)
- break;
- }
-
- if (str == NULL)
- header_idx = -1;
- else {
- hdr->header_offsets[0] = uint32_to_offset(offset);
- header_idx = i;
-
- size = strlen(str) + 1;
- nb_size = uint32_to_nbo(size);
-
- memcpy(mmap_base + offset, &nb_size, sizeof(nb_size));
- offset += sizeof(nb_size);
- memcpy(mmap_base + offset, str, size);
- offset += (size + 3) & ~3;
- }
-
- used_fields = 0;
- rec = cache->index->lookup(cache->index, 1);
- while (rec != NULL) {
- cache_rec = mail_cache_lookup(cache, rec, 0);
- if (cache_rec == NULL)
- rec->cache_offset = 0;
- else if (offset_to_uint32(cache_rec->next_offset) == 0) {
- /* just one unmodified block, copy it */
- size = nbo_to_uint32(cache_rec->size);
- i_assert(offset + size <= new_file_size);
-
- memcpy(mmap_base + offset, cache_rec, size);
- rec->cache_offset = uint32_to_offset(offset);
-
- size = (size + 3) & ~3;
- offset += size;
- } else {
- /* multiple blocks, sort them into buffer */
- t_push();
- cache_rec = mail_cache_compress_record(cache, rec,
- header_idx,
- &size);
- i_assert(offset + size <= new_file_size);
- memcpy(mmap_base + offset, cache_rec, size);
- used_fields |= cache_rec->fields;
- t_pop();
-
- rec->cache_offset = uint32_to_offset(offset);
- offset += size;
- }
-
- rec = cache->index->next(cache->index, rec);
- }
-
- /* update header */
- hdr->indexid = cache->index->indexid;
- hdr->sync_id = cache->sync_id = cache->index->cache_sync_id =
- ++cache->index->header->cache_sync_id;
- hdr->used_file_size = uint32_to_nbo(offset);
- hdr->used_fields = used_fields;
- hdr->field_usage_start = uint32_to_nbo(ioloop_time);
-
- /* write everything to disk */
- if (msync(mmap_base, offset, MS_SYNC) < 0)
- return mail_cache_set_syscall_error(cache, "msync()");
-
- if (munmap(mmap_base, new_file_size) < 0)
- return mail_cache_set_syscall_error(cache, "munmap()");
-
- if (fdatasync(fd) < 0)
- return mail_cache_set_syscall_error(cache, "fdatasync()");
- return TRUE;
-}
-
-int mail_cache_compress(struct mail_cache *cache)
-{
- int fd, ret = TRUE;
-
- i_assert(cache->trans_ctx == NULL);
-
- if (cache->anon_mmap)
- return TRUE;
-
- if (!cache->index->set_lock(cache->index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
-
- if (mail_cache_lock(cache, TRUE) <= 0)
- return FALSE;
-
-#ifdef DEBUG
- i_warning("Compressing cache file %s", cache->filepath);
-#endif
-
- fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
- MAIL_CACHE_LOCK_CHANGE_TIMEOUT,
- MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL);
- if (fd == -1) {
- mail_cache_set_syscall_error(cache, "file_dotlock_open()");
- return FALSE;
- }
-
- /* now we'll begin the actual moving. keep rebuild-flag on
- while doing it. */
- cache->index->header->flags |= MAIL_INDEX_HDR_FLAG_REBUILD;
- if (!mail_index_fmdatasync(cache->index, cache->index->header_size))
- return FALSE;
-
- if (!mail_cache_copy(cache, fd)) {
- (void)file_dotlock_delete(cache->filepath, fd);
- ret = FALSE;
- } else {
- mail_cache_file_close(cache);
- cache->fd = dup(fd);
-
- if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
- mail_cache_set_syscall_error(cache,
- "file_dotlock_replace()");
- ret = FALSE;
- }
-
- if (!mmap_update(cache, 0, 0))
- ret = FALSE;
- }
-
- /* headers could have changed, reread them */
- memset(cache->split_offsets, 0, sizeof(cache->split_offsets));
- memset(cache->split_headers, 0, sizeof(cache->split_headers));
-
- if (ret) {
- cache->index->header->flags &=
- ~(MAIL_INDEX_HDR_FLAG_REBUILD |
- MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE);
- }
-
- if (!mail_cache_unlock(cache))
- ret = FALSE;
-
- return ret;
-}
-
-int mail_cache_truncate(struct mail_cache *cache)
-{
- struct mail_cache_header hdr;
- int ret, fd;
-
- i_assert(cache->index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
- memset(&hdr, 0, sizeof(hdr));
- hdr.indexid = cache->index->indexid;
- hdr.sync_id = cache->sync_id = cache->index->cache_sync_id =
- ++cache->index->header->cache_sync_id;
- hdr.used_file_size = uint32_to_nbo(sizeof(hdr));
- cache->used_file_size = sizeof(hdr);
-
- if (cache->anon_mmap) {
- *cache->header = hdr;
- return TRUE;
- }
-
- ret = mail_cache_open_and_verify(cache, TRUE);
- if (ret != 0)
- return ret > 0;
-
- fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
- MAIL_CACHE_LOCK_CHANGE_TIMEOUT,
- MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL);
- if (fd == -1) {
- mail_cache_set_syscall_error(cache, "file_dotlock_open()");
- return FALSE;
- }
-
- if (write_full(fd, &hdr, sizeof(hdr)) < 0) {
- mail_cache_set_syscall_error(cache, "write_full()");
- (void)file_dotlock_delete(cache->filepath, fd);
- return FALSE;
- }
- if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) {
- mail_cache_set_syscall_error(cache, "file_set_size()");
- (void)file_dotlock_delete(cache->filepath, fd);
- return FALSE;
- }
-
- mail_cache_file_close(cache);
- cache->fd = dup(fd);
-
- if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
- mail_cache_set_syscall_error(cache, "file_dotlock_replace()");
- return FALSE;
- }
-
- cache->mmap_refresh = TRUE;
- if (!mmap_update(cache, 0, sizeof(struct mail_cache_header)))
- return FALSE;
-
- return TRUE;
-}
-
-int mail_cache_mark_file_deleted(struct mail_cache *cache)
-{
- uint32_t indexid = 0;
-
- if (cache->anon_mmap)
- cache->header->indexid = 0;
- else {
- if (pwrite(cache->fd, &indexid, sizeof(indexid), 0) < 0)
- return mail_cache_set_syscall_error(cache, "pwrite()");
- }
- return TRUE;
-}
-
-int mail_cache_lock(struct mail_cache *cache, int nonblock)
-{
- int ret;
-
- if (cache->locks++ != 0)
- return TRUE;
-
- if (cache->anon_mmap)
- return TRUE;
-
- if (nonblock) {
- ret = file_try_lock(cache->fd, F_WRLCK);
- if (ret < 0)
- mail_cache_set_syscall_error(cache, "file_try_lock()");
- } else {
- ret = file_wait_lock(cache->fd, F_WRLCK);
- if (ret <= 0)
- mail_cache_set_syscall_error(cache, "file_wait_lock()");
- }
-
- if (ret > 0) {
- if (!mmap_update(cache, 0, 0)) {
- (void)mail_cache_unlock(cache);
- return -1;
- }
- if (cache->sync_id != cache->index->cache_sync_id) {
- /* we have the cache file locked and sync_id still
- doesn't match. it means we crashed between updating
- cache file and updating sync_id in index header.
- just update the sync_ids so they match. */
- i_warning("Updating broken sync_id in cache file %s",
- cache->filepath);
- cache->sync_id = cache->header->sync_id =
- cache->index->cache_sync_id;
- }
- }
- return ret;
-}
-
-int mail_cache_unlock(struct mail_cache *cache)
-{
- if (--cache->locks > 0)
- return TRUE;
-
- if (cache->anon_mmap)
- return TRUE;
-
- if (file_wait_lock(cache->fd, F_UNLCK) <= 0) {
- mail_cache_set_syscall_error(cache, "file_wait_lock(F_UNLCK)");
- return FALSE;
- }
-
- return TRUE;
-}
-
-void mail_cache_unlock_later(struct mail_cache *cache)
-{
- cache->index->cache_later_locks++;
-}
-
-int mail_cache_is_locked(struct mail_cache *cache)
-{
- return cache->locks > 0;
-}
-
-int mail_cache_transaction_begin(struct mail_cache *cache, int nonblock,
- struct mail_cache_transaction_ctx **ctx_r)
-{
- int ret;
-
- i_assert(cache->trans_ctx == NULL);
-
- ret = mail_cache_lock(cache, nonblock);
- if (ret <= 0)
- return ret;
-
- *ctx_r = i_new(struct mail_cache_transaction_ctx, 1);
- (*ctx_r)->cache = cache;
- (*ctx_r)->cache_data =
- buffer_create_dynamic(system_pool, 8192, (size_t)-1);
- (*ctx_r)->last_idx = (unsigned int)-1;
-
- cache->trans_ctx = *ctx_r;
- return 1;
-}
-
-int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx)
-{
- int ret = TRUE;
-
- i_assert(ctx->cache->trans_ctx != NULL);
-
- (void)mail_cache_transaction_rollback(ctx);
-
- if (!mail_cache_unlock(ctx->cache))
- ret = FALSE;
-
- ctx->cache->trans_ctx = NULL;
-
- if (ctx->cache_marks != NULL)
- buffer_free(ctx->cache_marks);
- if (ctx->index_marks != NULL)
- buffer_free(ctx->index_marks);
- buffer_free(ctx->cache_data);
- i_free(ctx);
- return ret;
-}
-
-static void mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
-{
- memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec));
- ctx->last_idx = (unsigned int)-1;
-
- ctx->next_unused_header_lowwater = 0;
- ctx->first_uid = ctx->last_uid = ctx->prev_uid = 0;
- ctx->prev_fields = 0;
-
- if (ctx->cache_marks != NULL)
- buffer_set_used_size(ctx->cache_marks, 0);
- if (ctx->index_marks != NULL)
- buffer_set_used_size(ctx->index_marks, 0);
- buffer_set_used_size(ctx->cache_data, 0);
-}
-
-static void mark_update(buffer_t **buf, uint32_t offset, uint32_t data)
-{
- if (*buf == NULL)
- *buf = buffer_create_dynamic(system_pool, 1024, (size_t)-1);
-
- /* data is in big endian, we want to update only the lowest byte */
- buffer_append(*buf, &offset, sizeof(offset));
- buffer_append(*buf, &data, sizeof(data));
-}
-
-static int write_mark_updates(struct mail_index *index, buffer_t *marks,
- const char *path, int fd)
-{
- const uint32_t *data, *end;
- size_t size;
-
- data = buffer_get_data(marks, &size);
- end = data + size/sizeof(uint32_t);
-
- while (data < end) {
- if (pwrite(fd, data+1, sizeof(*data), data[0]) < 0) {
- index_file_set_syscall_error(index, path, "pwrite()");
- return FALSE;
- }
- data += 2;
- }
- return TRUE;
-}
-
-static void write_mark_updates_in_memory(buffer_t *marks, void *mmap_base,
- size_t mmap_length)
-{
- const unsigned char *data, *end;
- uint32_t offset;
- size_t size;
-
- data = buffer_get_data(marks, &size);
- end = data + size;
-
- while (data < end) {
- memcpy(&offset, data, sizeof(offset));
- data += sizeof(offset);
-
- i_assert(offset <= mmap_length - sizeof(uint32_t));
- memcpy((char *) mmap_base + offset, data, sizeof(uint32_t));
- data += sizeof(uint32_t);
- }
-}
-
-static void commit_all_changes_in_memory(struct mail_cache_transaction_ctx *ctx)
-{
- struct mail_cache *cache = ctx->cache;
-
- if (ctx->cache_marks != NULL) {
- write_mark_updates_in_memory(ctx->cache_marks,
- cache->mmap_base,
- cache->mmap_length);
- }
- if (ctx->index_marks != NULL) {
- write_mark_updates_in_memory(ctx->index_marks,
- cache->index->mmap_base,
- cache->index->mmap_used_length);
- }
-}
-
-static int commit_all_changes(struct mail_cache_transaction_ctx *ctx)
-{
- struct mail_cache *cache = ctx->cache;
- uint32_t cont;
-
- if (ctx->cache->anon_mmap) {
- commit_all_changes_in_memory(ctx);
- return TRUE;
- }
-
- /* write everything to disk */
- if (msync(cache->mmap_base, cache->mmap_length, MS_SYNC) < 0)
- return mail_cache_set_syscall_error(cache, "msync()");
-
- if (fdatasync(cache->fd) < 0)
- return mail_cache_set_syscall_error(cache, "fdatasync()");
-
- if (ctx->cache_marks != NULL &&
- buffer_get_used_size(ctx->cache_marks) != 0) {
- /* now that we're sure it's there, set on all the used-bits */
- if (!write_mark_updates(cache->index, ctx->cache_marks,
- cache->filepath, cache->fd))
- return FALSE;
-
- /* update continued records count */
- cont = nbo_to_uint32(cache->header->continued_record_count);
-
- cont += buffer_get_used_size(ctx->cache_marks) /
- (sizeof(uint32_t) * 2);
-
- if (cont * 100 / cache->index->header->messages_count >=
- COMPRESS_CONTINUED_PERCENTAGE &&
- cache->used_file_size >= COMPRESS_MIN_SIZE) {
- /* too many continued rows, compress */
- cache->index->set_flags |=
- MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
- }
-
- cache->header->continued_record_count = uint32_to_nbo(cont);
- }
-
- /* write index last */
- if (ctx->index_marks != NULL &&
- buffer_get_used_size(ctx->index_marks) != 0) {
- if (!mail_index_fmdatasync(cache->index,
- cache->index->mmap_used_length))
- return FALSE;
-
- if (!write_mark_updates(cache->index, ctx->index_marks,
- cache->index->filepath,
- cache->index->fd))
- return FALSE;
- }
- return TRUE;
-}
-
-int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx)
-{
- int ret = TRUE;
-
- if (ctx->last_idx != (unsigned int)-1) {
- if (!mail_cache_write(ctx))
- return FALSE;
- }
-
- ctx->cache->header->used_file_size =
- uint32_to_nbo(ctx->cache->used_file_size);
-
- if (!commit_all_changes(ctx))
- ret = FALSE;
-
- if (ctx->next_unused_header_lowwater == MAIL_CACHE_HEADERS_COUNT) {
- /* they're all used - compress the cache to get more */
- ctx->cache->index->set_flags |=
- MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
- }
-
- mail_cache_transaction_flush(ctx);
- return ret;
-}
-
-int mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx)
-{
- struct mail_cache *cache = ctx->cache;
- unsigned int i;
-
- /* no need to actually modify the file - we just didn't update
- used_file_size */
- cache->used_file_size = nbo_to_uint32(cache->header->used_file_size);
-
- /* make sure we don't cache the headers */
- for (i = 0; i < ctx->next_unused_header_lowwater; i++) {
- if (offset_to_uint32(cache->header->header_offsets[i]) == 0)
- cache->split_offsets[i] = 1;
- }
-
- mail_cache_transaction_flush(ctx);
- return TRUE;
-}
-
-static int mail_cache_grow(struct mail_cache *cache, uint32_t size)
-{
- struct stat st;
- void *base;
- uoff_t grow_size, new_fsize;
-
- new_fsize = cache->used_file_size + size;
- grow_size = new_fsize / 100 * MAIL_CACHE_GROW_PERCENTAGE;
- if (grow_size < 16384)
- grow_size = 16384;
-
- new_fsize += grow_size;
- new_fsize &= ~1023;
-
- if (cache->anon_mmap) {
- i_assert(new_fsize < SSIZE_T_MAX);
-
- base = mremap_anon(cache->mmap_base, cache->mmap_length,
- (size_t)new_fsize, MREMAP_MAYMOVE);
- if (base == MAP_FAILED) {
- mail_cache_set_syscall_error(cache, "mremap_anon()");
- return FALSE;
- }
-
- cache->mmap_base = base;
- cache->mmap_length = (size_t)new_fsize;
- cache->header = cache->mmap_base;
- return TRUE;
- }
-
- if (fstat(cache->fd, &st) < 0)
- return mail_cache_set_syscall_error(cache, "fstat()");
-
- if (cache->used_file_size + size <= (uoff_t)st.st_size) {
- /* no need to grow, just update mmap */
- if (!mmap_update(cache, 0, 0))
- return FALSE;
-
- i_assert(cache->mmap_length >= (uoff_t)st.st_size);
- return TRUE;
- }
-
- if (st.st_size < (off_t)sizeof(struct mail_cache_header))
- return mail_cache_set_corrupted(cache, "Header is missing");
-
- if (file_set_size(cache->fd, (off_t)new_fsize) < 0)
- return mail_cache_set_syscall_error(cache, "file_set_size()");
-
- return mmap_update(cache, 0, 0);
-}
-
-static uint32_t mail_cache_append_space(struct mail_cache_transaction_ctx *ctx,
- uint32_t size)
-{
- /* NOTE: must be done within transaction or rollback would break it */
- uint32_t offset;
-
- i_assert((size & 3) == 0);
-
- offset = ctx->cache->used_file_size;
- if (offset >= 0x40000000) {
- index_set_error(ctx->cache->index, "Cache file too large: %s",
- ctx->cache->filepath);
- return 0;
- }
-
- if (offset + size > ctx->cache->mmap_length) {
- if (!mail_cache_grow(ctx->cache, size))
- return 0;
- }
-
- ctx->cache->used_file_size += size;
- return offset;
-}
-
-static const char *
-mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx)
-{
- uint32_t offset, data_size;
- unsigned char *buf;
-
- offset = offset_to_uint32(cache->header->header_offsets[idx]);
-
- if (offset == 0)
- return NULL;
-
- if (!mmap_update(cache, offset, 1024))
- return NULL;
-
- if (offset + sizeof(data_size) > cache->mmap_length) {
- mail_cache_set_corrupted(cache, "Header %u points outside file",
- idx);
- return NULL;
- }
-
- buf = cache->mmap_base;
- memcpy(&data_size, buf + offset, sizeof(data_size));
- data_size = nbo_to_uint32(data_size);
- offset += sizeof(data_size);
-
- if (data_size == 0) {
- mail_cache_set_corrupted(cache,
- "Header %u points to empty string", idx);
- return NULL;
- }
-
- if (!mmap_update(cache, offset, data_size))
- return NULL;
-
- if (offset + data_size > cache->mmap_length) {
- mail_cache_set_corrupted(cache, "Header %u points outside file",
- idx);
- return NULL;
- }
-
- buf = cache->mmap_base;
- if (buf[offset + data_size - 1] != '\0') {
- mail_cache_set_corrupted(cache,
- "Header %u points to invalid string", idx);
- return NULL;
- }
-
- return buf + offset;
-}
-
-static const char *const *
-split_header(struct mail_cache *cache, const char *header)
-{
- const char *const *arr, *const *tmp;
- const char *null = NULL;
- char *str;
- buffer_t *buf;
-
- if (header == NULL)
- return NULL;
-
- arr = t_strsplit(header, "\n");
- buf = buffer_create_dynamic(cache->split_header_pool, 32, (size_t)-1);
- for (tmp = arr; *tmp != NULL; tmp++) {
- str = p_strdup(cache->split_header_pool, *tmp);
- buffer_append(buf, &str, sizeof(str));
- }
- buffer_append(buf, &null, sizeof(null));
-
- return buffer_get_data(buf, NULL);
-}
-
-const char *const *mail_cache_get_header_fields(struct mail_cache *cache,
- unsigned int idx)
-{
- const char *str;
- int i;
-
- i_assert(idx < MAIL_CACHE_HEADERS_COUNT);
-
- /* t_strsplit() is a bit slow, so we cache it */
- if (cache->header->header_offsets[idx] != cache->split_offsets[idx]) {
- p_clear(cache->split_header_pool);
-
- t_push();
- for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) {
- cache->split_offsets[i] =
- cache->header->header_offsets[i];
-
- str = mail_cache_get_header_fields_str(cache, i);
- cache->split_headers[i] = split_header(cache, str);
- }
- t_pop();
- }
-
- return cache->split_headers[idx];
-}
-
-static const char *write_header_string(const char *const headers[],
- uint32_t *size_r)
-{
- buffer_t *buffer;
- size_t size;
-
- buffer = buffer_create_dynamic(pool_datastack_create(),
- 512, (size_t)-1);
-
- while (*headers != NULL) {
- if (buffer_get_used_size(buffer) != 0)
- buffer_append(buffer, "\n", 1);
- buffer_append(buffer, *headers, strlen(*headers));
- headers++;
- }
- buffer_append(buffer, null4, 1);
-
- size = buffer_get_used_size(buffer);
- if ((size & 3) != 0) {
- buffer_append(buffer, null4, 4 - (size & 3));
- size += 4 - (size & 3);
- }
- *size_r = size;
- return buffer_get_data(buffer, NULL);
-}
-
-int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx,
- unsigned int idx, const char *const headers[])
-{
- struct mail_cache *cache = ctx->cache;
- uint32_t offset, update_offset, size;
- const char *header_str, *prev_str;
-
- i_assert(*headers != NULL);
- i_assert(idx < MAIL_CACHE_HEADERS_COUNT);
- i_assert(idx >= ctx->next_unused_header_lowwater);
- i_assert(offset_to_uint32(cache->header->header_offsets[idx]) == 0);
-
- t_push();
-
- header_str = write_header_string(headers, &size);
- if (idx != 0) {
- prev_str = mail_cache_get_header_fields_str(cache, idx-1);
- if (prev_str == NULL) {
- t_pop();
- return FALSE;
- }
-
- i_assert(strcmp(header_str, prev_str) != 0);
- }
-
- offset = mail_cache_append_space(ctx, size + sizeof(uint32_t));
- if (offset != 0) {
- memcpy((char *) cache->mmap_base + offset + sizeof(uint32_t),
- header_str, size);
-
- size = uint32_to_nbo(size);
- memcpy((char *) cache->mmap_base + offset,
- &size, sizeof(uint32_t));
-
- /* update cached headers */
- cache->split_offsets[idx] = cache->header->header_offsets[idx];
- cache->split_headers[idx] = split_header(cache, header_str);
-
- /* mark used-bit to be updated later. not really needed for
- read-safety, but if transaction get rolled back we can't let
- this point to invalid location. */
- update_offset = (char *) &cache->header->header_offsets[idx] -
- (char *) cache->mmap_base;
- mark_update(&ctx->cache_marks, update_offset,
- uint32_to_offset(offset));
-
- /* make sure get_header_fields() still works for this header
- while the transaction isn't yet committed. */
- ctx->next_unused_header_lowwater = idx + 1;
- }
-
- t_pop();
- return offset > 0;
-}
-
-static struct mail_cache_record *
-cache_get_record(struct mail_cache *cache, uint32_t offset)
-{
-#define CACHE_PREFETCH 1024
- struct mail_cache_record *cache_rec;
- size_t size;
-
- offset = offset_to_uint32(offset);
- if (offset == 0)
- return NULL;
-
- if (!mmap_update(cache, offset, sizeof(*cache_rec) + CACHE_PREFETCH))
- return NULL;
-
- if (offset + sizeof(*cache_rec) > cache->mmap_length) {
- mail_cache_set_corrupted(cache, "record points outside file");
- return NULL;
- }
- cache_rec = CACHE_RECORD(cache, offset);
-
- size = nbo_to_uint32(cache_rec->size);
- if (size < sizeof(*cache_rec)) {
- mail_cache_set_corrupted(cache, "invalid record size");
- return NULL;
- }
- if (size > CACHE_PREFETCH) {
- if (!mmap_update(cache, offset, size))
- return NULL;
- }
-
- if (offset + size > cache->mmap_length) {
- mail_cache_set_corrupted(cache, "record points outside file");
- return NULL;
- }
- return cache_rec;
-}
-
-static struct mail_cache_record *
-cache_get_next_record(struct mail_cache *cache, struct mail_cache_record *rec)
-{
- struct mail_cache_record *next;
-
- next = cache_get_record(cache, rec->next_offset);
- if (next != NULL && next <= rec) {
- mail_cache_set_corrupted(cache, "next_offset points backwards");
- return NULL;
- }
- return next;
-}
-
-static int mail_cache_write(struct mail_cache_transaction_ctx *ctx)
-{
- struct mail_cache *cache = ctx->cache;
- struct mail_cache_record *cache_rec, *next;
- struct mail_index_record *rec;
- uint32_t write_offset, update_offset;
- const void *buf;
- size_t size, buf_size;
-
- buf = buffer_get_data(ctx->cache_data, &buf_size);
-
- size = sizeof(*cache_rec) + buf_size;
- ctx->cache_rec.size = uint32_to_nbo(size);
-
- write_offset = mail_cache_append_space(ctx, size);
- if (write_offset == 0)
- return FALSE;
-
- rec = INDEX_RECORD_AT(ctx->cache->index, ctx->last_idx);
- ctx->last_idx = (unsigned int)-1;
-
- cache_rec = cache_get_record(cache, rec->cache_offset);
- if (cache_rec == NULL) {
- /* first cache record - update offset in index file */
- i_assert(cache->index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
- /* mark cache_offset to be updated later */
- update_offset = (char *) &rec->cache_offset -
- (char *) cache->index->mmap_base;
- mark_update(&ctx->index_marks, update_offset,
- uint32_to_offset(write_offset));
- } else {
- /* find the last cache record */
- while ((next = cache_get_next_record(cache, cache_rec)) != NULL)
- cache_rec = next;
-
- /* mark next_offset to be updated later */
- update_offset = (char *) &cache_rec->next_offset -
- (char *) cache->mmap_base;
- mark_update(&ctx->cache_marks, update_offset,
- uint32_to_offset(write_offset));
- }
-
- memcpy((char *) cache->mmap_base + write_offset,
- &ctx->cache_rec, sizeof(ctx->cache_rec));
- memcpy((char *) cache->mmap_base + write_offset +
- sizeof(ctx->cache_rec), buf, buf_size);
-
- /* reset the write context */
- memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec));
- buffer_set_used_size(ctx->cache_data, 0);
- return TRUE;
-}
-
-static struct mail_cache_record *
-mail_cache_lookup(struct mail_cache *cache, const struct mail_index_record *rec,
- enum mail_cache_field fields)
-{
- struct mail_cache_record *cache_rec;
- unsigned int idx;
-
- if (cache->trans_ctx != NULL &&
- cache->trans_ctx->first_uid <= rec->uid &&
- cache->trans_ctx->last_uid >= rec->uid &&
- (cache->trans_ctx->prev_uid != rec->uid || fields == 0 ||
- (cache->trans_ctx->prev_fields & fields) != 0)) {
- /* we have to auto-commit since we're not capable of looking
- into uncommitted records. it would be possible by checking
- index_marks and cache_marks, but it's just more trouble
- than worth. */
- idx = INDEX_RECORD_INDEX(cache->index, rec);
- if (cache->trans_ctx->last_idx == idx) {
- if (!mail_cache_write(cache->trans_ctx))
- return NULL;
- }
-
- if (!mail_cache_transaction_commit(cache->trans_ctx))
- return NULL;
- }
-
- cache_rec = cache_get_record(cache, rec->cache_offset);
- if (cache_rec == NULL)
- return NULL;
-
- return cache_rec;
-}
-
-static int get_field_num(enum mail_cache_field field)
-{
- unsigned int mask;
- int i;
-
- for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
- if ((field & mask) != 0)
- return i;
- }
-
- return -1;
-}
-
-static size_t get_insert_offset(struct mail_cache_transaction_ctx *ctx,
- enum mail_cache_field field)
-{
- const unsigned char *buf;
- unsigned int mask;
- uint32_t data_size;
- size_t offset = 0;
- int i;
-
- buf = buffer_get_data(ctx->cache_data, NULL);
-
- for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
- if ((field & mask) != 0)
- return offset;
-
- if ((ctx->cache_rec.fields & mask) != 0) {
- if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
- data_size = mail_cache_field_sizes[i];
- else {
- memcpy(&data_size, buf + offset,
- sizeof(data_size));
- data_size = nbo_to_uint32(data_size);
- offset += sizeof(data_size);
- }
- offset += (data_size + 3) & ~3;
- }
- }
-
- i_unreached();
- return offset;
-}
-
-int mail_cache_add(struct mail_cache_transaction_ctx *ctx,
- struct mail_index_record *rec, enum mail_cache_field field,
- const void *data, size_t data_size)
-{
- uint32_t nb_data_size;
- size_t full_size, offset;
- unsigned char *buf;
- unsigned int idx;
- int field_num;
-
- i_assert(data_size > 0);
- i_assert(data_size < (uint32_t)-1);
-
- nb_data_size = uint32_to_nbo((uint32_t)data_size);
-
- if ((field & MAIL_CACHE_FIXED_MASK) != 0) {
- field_num = get_field_num(field);
- i_assert(field_num != -1);
- i_assert(mail_cache_field_sizes[field_num] == data_size);
- } else if ((field & MAIL_CACHE_STRING_MASK) != 0) {
- i_assert(((char *) data)[data_size-1] == '\0');
- }
-
- /* NOTE: we use index because the record pointer might not last. */
- idx = INDEX_RECORD_INDEX(ctx->cache->index, rec);
- if (ctx->last_idx != idx && ctx->last_idx != (unsigned int)-1) {
- if (!mail_cache_write(ctx))
- return FALSE;
- }
- ctx->last_idx = idx;
-
- i_assert((ctx->cache_rec.fields & field) == 0);
-
- full_size = (data_size + 3) & ~3;
- if ((field & MAIL_CACHE_FIXED_MASK) == 0)
- full_size += sizeof(nb_data_size);
-
- /* fields must be ordered. find where to insert it. */
- if (field > ctx->cache_rec.fields)
- buf = buffer_append_space_unsafe(ctx->cache_data, full_size);
- else {
- offset = get_insert_offset(ctx, field);
- buffer_copy(ctx->cache_data, offset + full_size,
- ctx->cache_data, offset, (size_t)-1);
- buf = buffer_get_space_unsafe(ctx->cache_data,
- offset, full_size);
- }
- ctx->cache_rec.fields |= field;
-
- /* @UNSAFE */
- if ((field & MAIL_CACHE_FIXED_MASK) == 0) {
- memcpy(buf, &nb_data_size, sizeof(nb_data_size));
- buf += sizeof(nb_data_size);
- }
- memcpy(buf, data, data_size); buf += data_size;
- if ((data_size & 3) != 0)
- memset(buf, 0, 4 - (data_size & 3));
-
- /* remember the transaction uid range */
- if (rec->uid < ctx->first_uid || ctx->first_uid == 0)
- ctx->first_uid = rec->uid;
- if (rec->uid > ctx->last_uid)
- ctx->last_uid = rec->uid;
-
- if (ctx->prev_uid != rec->uid) {
- ctx->prev_uid = rec->uid;
- ctx->prev_fields = 0;
- }
- ctx->prev_fields |= field;
-
- return TRUE;
-}
-
-int mail_cache_delete(struct mail_cache_transaction_ctx *ctx,
- struct mail_index_record *rec)
-{
- struct mail_cache *cache = ctx->cache;
- struct mail_cache_record *cache_rec;
- uint32_t deleted_space;
- uoff_t max_del_space;
-
- cache_rec = mail_cache_lookup(cache, rec, 0);
- if (cache_rec == NULL)
- return TRUE;
-
- /* NOTE: it would be nice to erase the cached data for the record,
- but some other processes might still be using them. So, we just
- update the deleted_space in header */
- deleted_space = nbo_to_uint32(cache->header->deleted_space);
-
- do {
- deleted_space -= nbo_to_uint32(cache_rec->size);
- cache_rec = cache_get_next_record(cache, cache_rec);
- } while (cache_rec != NULL);
-
- /* see if we've reached the max. deleted space in file */
- max_del_space = cache->used_file_size / 100 * COMPRESS_PERCENTAGE;
- if (deleted_space >= max_del_space &&
- cache->used_file_size >= COMPRESS_MIN_SIZE)
- cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
-
- cache->header->deleted_space = uint32_to_nbo(deleted_space);
-
- return TRUE;
-}
-
-enum mail_cache_field
-mail_cache_get_fields(struct mail_cache *cache,
- const struct mail_index_record *rec)
-{
- struct mail_cache_record *cache_rec;
- enum mail_cache_field fields = 0;
-
- cache_rec = mail_cache_lookup(cache, rec, 0);
- while (cache_rec != NULL) {
- fields |= cache_rec->fields;
- cache_rec = cache_get_next_record(cache, cache_rec);
- }
-
- return fields;
-}
-
-static int cache_get_field(struct mail_cache *cache,
- struct mail_cache_record *cache_rec,
- enum mail_cache_field field,
- void **data_r, size_t *size_r)
-{
- unsigned char *buf;
- unsigned int mask;
- uint32_t rec_size, data_size;
- size_t offset, next_offset;
- int i;
-
- rec_size = nbo_to_uint32(cache_rec->size);
- buf = (unsigned char *) cache_rec;
- offset = sizeof(*cache_rec);
-
- for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
- if ((cache_rec->fields & mask) == 0)
- continue;
-
- /* all records are at least 32bit. we have to check this
- before getting data_size. */
- if (offset + sizeof(uint32_t) > rec_size) {
- mail_cache_set_corrupted(cache,
- "Record continues outside it's allocated size");
- return FALSE;
- }
-
- if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
- data_size = mail_cache_field_sizes[i];
- else {
- memcpy(&data_size, buf + offset, sizeof(data_size));
- data_size = nbo_to_uint32(data_size);
- offset += sizeof(data_size);
- }
-
- next_offset = offset + ((data_size + 3) & ~3);
- if (next_offset > rec_size) {
- mail_cache_set_corrupted(cache,
- "Record continues outside it's allocated size");
- return FALSE;
- }
-
- if (field == mask) {
- if (data_size == 0) {
- mail_cache_set_corrupted(cache,
- "Field size is 0");
- return FALSE;
- }
- *data_r = buf + offset;
- *size_r = data_size;
- return TRUE;
- }
- offset = next_offset;
- }
-
- i_unreached();
- return FALSE;
-}
-
-static int cache_lookup_field(struct mail_cache *cache,
- const struct mail_index_record *rec,
- enum mail_cache_field field,
- void **data_r, size_t *size_r)
-{
- struct mail_cache_record *cache_rec;
-
- cache_rec = mail_cache_lookup(cache, rec, field);
- while (cache_rec != NULL) {
- if ((cache_rec->fields & field) != 0) {
- return cache_get_field(cache, cache_rec, field,
- data_r, size_r);
- }
- cache_rec = cache_get_next_record(cache, cache_rec);
- }
-
- return FALSE;
-}
-
-int mail_cache_lookup_field(struct mail_cache *cache,
- const struct mail_index_record *rec,
- enum mail_cache_field field,
- const void **data_r, size_t *size_r)
-{
- void *data;
-
- if (!cache_lookup_field(cache, rec, field, &data, size_r))
- return FALSE;
-
- *data_r = data;
- return TRUE;
-}
-
-const char *mail_cache_lookup_string_field(struct mail_cache *cache,
- const struct mail_index_record *rec,
- enum mail_cache_field field)
-{
- const void *data;
- size_t size;
-
- i_assert((field & MAIL_CACHE_STRING_MASK) != 0);
-
- if (!mail_cache_lookup_field(cache, rec, field, &data, &size))
- return NULL;
-
- if (((const char *) data)[size-1] != '\0') {
- mail_cache_set_corrupted(cache,
- "String field %x doesn't end with NUL", field);
- return NULL;
- }
- return data;
-}
-
-int mail_cache_copy_fixed_field(struct mail_cache *cache,
- const struct mail_index_record *rec,
- enum mail_cache_field field,
- void *buffer, size_t buffer_size)
-{
- const void *data;
- size_t size;
-
- i_assert((field & MAIL_CACHE_FIXED_MASK) != 0);
-
- if (!mail_cache_lookup_field(cache, rec, field, &data, &size))
- return FALSE;
-
- if (buffer_size != size) {
- i_panic("cache: fixed field %x wrong size "
- "(%"PRIuSIZE_T" vs %"PRIuSIZE_T")",
- field, size, buffer_size);
- }
-
- memcpy(buffer, data, buffer_size);
- return TRUE;
-}
-
-void mail_cache_mark_missing(struct mail_cache *cache,
- enum mail_cache_field fields)
-{
- // FIXME: count these
-}
-
-enum mail_index_record_flag
-mail_cache_get_index_flags(struct mail_cache *cache,
- const struct mail_index_record *rec)
-{
- enum mail_index_record_flag flags;
-
- if (!mail_cache_copy_fixed_field(cache, rec, MAIL_CACHE_INDEX_FLAGS,
- &flags, sizeof(flags)))
- return 0;
-
- return flags;
-}
-
-int mail_cache_update_index_flags(struct mail_cache *cache,
- struct mail_index_record *rec,
- enum mail_index_record_flag flags)
-{
- void *data;
- size_t size;
-
- i_assert(cache->locks > 0);
-
- if (!cache_lookup_field(cache, rec, MAIL_CACHE_INDEX_FLAGS,
- &data, &size)) {
- mail_cache_set_corrupted(cache,
- "Missing index flags for record %u", rec->uid);
- return FALSE;
- }
-
- memcpy(data, &flags, sizeof(flags));
- return TRUE;
-}
-
-int mail_cache_update_location_offset(struct mail_cache *cache,
- struct mail_index_record *rec,
- uoff_t offset)
-{
- void *data;
- size_t size;
-
- i_assert(cache->locks > 0);
-
- if (!cache_lookup_field(cache, rec, MAIL_CACHE_LOCATION_OFFSET,
- &data, &size)) {
- mail_cache_set_corrupted(cache,
- "Missing location offset for record %u", rec->uid);
- return FALSE;
- }
-
- memcpy(data, &offset, sizeof(offset));
- return TRUE;
-}
-
-void *mail_cache_get_mmaped(struct mail_cache *cache, size_t *size)
-{
- if (!mmap_update(cache, 0, 0))
- return NULL;
-
- *size = cache->mmap_length;
- return cache->mmap_base;
-}
-
-int mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...)
-{
- va_list va;
-
- mail_cache_mark_file_deleted(cache);
- cache->index->inconsistent = TRUE; /* easiest way to rebuild */
-
- if (cache->silent)
- return FALSE;
-
- va_start(va, fmt);
- t_push();
- index_set_error(cache->index, "Corrupted index cache file %s: %s",
- cache->filepath, t_strdup_vprintf(fmt, va));
- t_pop();
- va_end(va);
-
- return FALSE;
-}
+++ /dev/null
-#ifndef __MAIL_CACHE_H
-#define __MAIL_CACHE_H
-
-#include "mail-index.h"
-
-#define MAIL_CACHE_FILE_PREFIX ".cache"
-
-#define MAIL_CACHE_HEADERS_COUNT 4
-
-struct mail_cache_transaction_ctx;
-
-enum mail_cache_field {
- /* fixed size fields */
- MAIL_CACHE_INDEX_FLAGS = 0x00000001,
- MAIL_CACHE_LOCATION_OFFSET = 0x00000002,
- MAIL_CACHE_MD5 = 0x00000004,
- MAIL_CACHE_SENT_DATE = 0x00000008,
- MAIL_CACHE_RECEIVED_DATE = 0x00000010,
- MAIL_CACHE_VIRTUAL_FULL_SIZE = 0x00000020,
- MAIL_CACHE_PHYSICAL_BODY_SIZE = 0x00000040,
-
- /* variable sized field */
- MAIL_CACHE_HEADERS1 = 0x40000000,
- MAIL_CACHE_HEADERS2 = 0x20000000,
- MAIL_CACHE_HEADERS3 = 0x10000000,
- MAIL_CACHE_HEADERS4 = 0x08000000,
- MAIL_CACHE_LOCATION = 0x04000000,
- MAIL_CACHE_BODY = 0x02000000,
- MAIL_CACHE_BODYSTRUCTURE = 0x01000000,
- MAIL_CACHE_ENVELOPE = 0x00800000,
- MAIL_CACHE_MESSAGEPART = 0x00400000,
-
- MAIL_CACHE_FIXED_MASK = MAIL_CACHE_INDEX_FLAGS |
- MAIL_CACHE_LOCATION_OFFSET |
- MAIL_CACHE_MD5 |
- MAIL_CACHE_SENT_DATE |
- MAIL_CACHE_RECEIVED_DATE |
- MAIL_CACHE_VIRTUAL_FULL_SIZE |
- MAIL_CACHE_PHYSICAL_BODY_SIZE,
- MAIL_CACHE_HEADERS_MASK = MAIL_CACHE_HEADERS1 |
- MAIL_CACHE_HEADERS2 |
- MAIL_CACHE_HEADERS3 |
- MAIL_CACHE_HEADERS4,
- MAIL_CACHE_STRING_MASK = MAIL_CACHE_HEADERS_MASK |
- MAIL_CACHE_LOCATION |
- MAIL_CACHE_BODY |
- MAIL_CACHE_BODYSTRUCTURE |
- MAIL_CACHE_ENVELOPE,
- MAIL_CACHE_BODYSTRUCTURE_MASK = MAIL_CACHE_BODY |
- MAIL_CACHE_BODYSTRUCTURE |
- MAIL_CACHE_MESSAGEPART
-};
-
-struct mail_sent_date {
- time_t time;
- int32_t timezone;
-};
-
-extern enum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT];
-
-int mail_cache_open_or_create(struct mail_index *index);
-void mail_cache_free(struct mail_cache *cache);
-
-void mail_cache_set_defaults(struct mail_cache *cache,
- enum mail_cache_field default_cache_fields,
- enum mail_cache_field never_cache_fields);
-
-/* Compress cache file. */
-int mail_cache_compress(struct mail_cache *cache);
-
-/* Truncate the cache file and update it's indexid */
-int mail_cache_truncate(struct mail_cache *cache);
-
-/* Set indexid to 0 to notify other processes using this file that they should
- re-open it. */
-int mail_cache_mark_file_deleted(struct mail_cache *cache);
-
-/* Explicitly lock the cache file. Returns 1 if ok, 0 if nonblock is TRUE and
- we couldn't immediately get a lock, or -1 if error. */
-int mail_cache_lock(struct mail_cache *cache, int nonblock);
-int mail_cache_unlock(struct mail_cache *cache);
-
-/* Mark the lock to be removed when unlocking index file. */
-void mail_cache_unlock_later(struct mail_cache *cache);
-
-/* Returns TRUE if cache file is locked. */
-int mail_cache_is_locked(struct mail_cache *cache);
-
-/* Begin transaction. Returns same as mail_cache_lock(). Note that if you
- call lookup functions for messages within first and last message in
- transaction, the transaction will be automatically committed. */
-int mail_cache_transaction_begin(struct mail_cache *cache, int nonblock,
- struct mail_cache_transaction_ctx **ctx_r);
-/* End transaction. Single transaction can have multiple commits/rollbacks.
- If there's any pending changes, they will be rolled back. */
-int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx);
-
-int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx);
-int mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx);
-
-/* Return NULL-terminated list of headers for given index, or NULL if
- header index isn't used. */
-const char *const *mail_cache_get_header_fields(struct mail_cache *cache,
- unsigned int idx);
-/* Set list of headers for given index. */
-int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx,
- unsigned int idx, const char *const headers[]);
-
-/* Add new field to given record. Updates are not allowed. Fixed size fields
- must be exactly the expected size and they're converted to network byte
- order in disk. */
-int mail_cache_add(struct mail_cache_transaction_ctx *ctx,
- struct mail_index_record *rec, enum mail_cache_field field,
- const void *data, size_t data_size);
-
-/* Mark the given record deleted. */
-int mail_cache_delete(struct mail_cache_transaction_ctx *ctx,
- struct mail_index_record *rec);
-
-/* Return all fields that are currently cached for record. */
-enum mail_cache_field
-mail_cache_get_fields(struct mail_cache *cache,
- const struct mail_index_record *rec);
-
-/* Set data_r and size_r to point to wanted field in cache file.
- Returns TRUE if field was found. If field contains multiple fields,
- first one found is returned. This is mostly useful for finding headers. */
-int mail_cache_lookup_field(struct mail_cache *cache,
- const struct mail_index_record *rec,
- enum mail_cache_field field,
- const void **data_r, size_t *size_r);
-
-/* Return string field. */
-const char *mail_cache_lookup_string_field(struct mail_cache *cache,
- const struct mail_index_record *rec,
- enum mail_cache_field field);
-
-
-/* Copy fixed size field to given buffer. buffer_size must be exactly the
- expected size. The result will be converted to host byte order.
- Returns TRUE if field was found. */
-int mail_cache_copy_fixed_field(struct mail_cache *cache,
- const struct mail_index_record *rec,
- enum mail_cache_field field,
- void *buffer, size_t buffer_size);
-
-/* Mark given fields as missing, ie. they should be cached when possible. */
-void mail_cache_mark_missing(struct mail_cache *cache,
- enum mail_cache_field fields);
-
-/* Return index flags. */
-enum mail_index_record_flag
-mail_cache_get_index_flags(struct mail_cache *cache,
- const struct mail_index_record *rec);
-
-/* Update index flags. The cache file must be locked and the flags must be
- already inserted to the record. */
-int mail_cache_update_index_flags(struct mail_cache *cache,
- struct mail_index_record *rec,
- enum mail_index_record_flag flags);
-
-/* Update location offset. External locking is assumed to take care of locking
- readers out to prevent race conditions. */
-int mail_cache_update_location_offset(struct mail_cache *cache,
- struct mail_index_record *rec,
- uoff_t offset);
-
-/* Return the whole file mmaped. */
-void *mail_cache_get_mmaped(struct mail_cache *cache, size_t *size);
-
-/* "Error in index cache file %s: ...". */
-int mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...)
- __attr_format__(2, 3);
-
-#endif
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "file-lock.h"
-#include "mmap-util.h"
-#include "write-full.h"
-#include "imap-util.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-#include "mail-custom-flags.h"
-
-#include <ctype.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-
-/* Header is simply a counter which is increased every time the file is
- updated. This allows other processes to easily notice if there's been
- any changes. */
-
-#define COUNTER_SIZE 4
-#define HEADER_SIZE (COUNTER_SIZE + 1) /* 0000\n */
-
-struct mail_custom_flags {
- struct mail_index *index;
- char *filepath;
- int fd;
- int lock_type;
-
- char sync_counter[COUNTER_SIZE];
- char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT];
-
- void *mmap_base;
- size_t mmap_length;
-
- unsigned int syncing:1;
- unsigned int noupdate:1;
- unsigned int changed:1;
-};
-
-static int lock_file(struct mail_custom_flags *mcf, int type);
-
-static int index_cf_set_syscall_error(struct mail_custom_flags *mcf,
- const char *function)
-{
- i_assert(function != NULL);
-
- if (ENOSPACE(errno)) {
- mcf->index->nodiskspace = TRUE;
- return FALSE;
- }
-
- index_set_error(mcf->index, "%s failed with custom flags file %s: %m",
- function, mcf->filepath);
- return FALSE;
-}
-
-static int update_mmap(struct mail_custom_flags *mcf)
-{
- if (mcf->mmap_base != NULL) {
- if (munmap(mcf->mmap_base, mcf->mmap_length) < 0)
- index_cf_set_syscall_error(mcf, "munmap()");
- }
-
-
- mcf->mmap_base = mcf->noupdate ?
- mmap_ro_file(mcf->fd, &mcf->mmap_length) :
- mmap_rw_file(mcf->fd, &mcf->mmap_length);
- if (mcf->mmap_base == MAP_FAILED) {
- mcf->mmap_base = NULL;
- return index_cf_set_syscall_error(mcf, "mmap()");
- }
-
- (void)madvise(mcf->mmap_base, mcf->mmap_length, MADV_SEQUENTIAL);
- return TRUE;
-}
-
-static int custom_flags_init(struct mail_custom_flags *mcf)
-{
- static char buf[HEADER_SIZE] = "0000\n";
- struct stat st;
- int failed;
-
- if (!lock_file(mcf, F_WRLCK))
- return FALSE;
-
- failed = FALSE;
-
- /* make sure it's still empty after locking */
- if (fstat(mcf->fd, &st) < 0) {
- index_cf_set_syscall_error(mcf, "fstat()");
- failed = TRUE;
- } else if (st.st_size < HEADER_SIZE) {
- /* write the header - it's a 4 byte counter as hex */
- if (write_full(mcf->fd, buf, HEADER_SIZE) < 0) {
- index_cf_set_syscall_error(mcf, "write_full()");
- failed = TRUE;
- }
- }
-
- if (!lock_file(mcf, F_UNLCK))
- return FALSE;
-
- return !failed;
-}
-
-static void custom_flags_sync(struct mail_custom_flags *mcf)
-{
- char *data, *data_end, *line;
- unsigned int num;
- int i;
-
- if (mcf->noupdate)
- return;
-
- memcpy(mcf->sync_counter, mcf->mmap_base, COUNTER_SIZE);
-
- for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
- if (mcf->custom_flags[i] != NULL) {
- i_free(mcf->custom_flags[i]);
- mcf->custom_flags[i] = NULL;
- }
- }
-
- data = mcf->mmap_base;
- data_end = data + mcf->mmap_length;
-
- /* this loop skips the first line, which is the header */
- while (data != data_end) {
- if (*data != '\n') {
- data++;
- continue;
- }
-
- /* beginning of line, get the index */
- if (data+1 == data_end)
- break;
- data++;
-
- if (!i_isdigit(*data))
- continue;
-
- num = 0;
- while (data != data_end && *data >= '0' && *data <= '9') {
- num = num*10 + (*data-'0');
- data++;
- }
-
- if (num < MAIL_CUSTOM_FLAGS_COUNT) {
- /* get the name */
- if (data == data_end || *data != ' ')
- continue;
-
- line = ++data;
- while (data != data_end && *data != '\n')
- data++;
-
- if (mcf->custom_flags[num] != NULL) {
- i_warning("Error in custom flags file %s: "
- "Duplicated ID %u", mcf->filepath,
- num);
- i_free(mcf->custom_flags[num]);
- }
-
- mcf->custom_flags[num] = i_strdup_until(line, data);
- }
- }
-}
-
-static int custom_flags_check_sync(struct mail_custom_flags *mcf)
-{
- if (mcf->fd == -1)
- return TRUE;
-
- if (mcf->mmap_length != 0 &&
- memcmp(mcf->sync_counter, mcf->mmap_base, COUNTER_SIZE) == 0)
- return TRUE;
-
- /* file modified, resync */
- if (!update_mmap(mcf))
- return FALSE;
-
- if (mcf->mmap_length < HEADER_SIZE && !mcf->noupdate) {
- /* it's broken, rewrite header */
- if (mcf->lock_type == F_RDLCK)
- (void)lock_file(mcf, F_UNLCK);
-
- if (lseek(mcf->fd, 0, SEEK_SET) < 0) {
- index_cf_set_syscall_error(mcf, "lseek()");
- return FALSE;
- }
-
- if (!custom_flags_init(mcf))
- return FALSE;
-
- if (!update_mmap(mcf))
- return FALSE;
- }
-
- custom_flags_sync(mcf);
- mcf->changed = TRUE;
- return TRUE;
-}
-
-static int lock_file(struct mail_custom_flags *mcf, int type)
-{
- if (mcf->lock_type == type)
- return TRUE;
-
- if (mcf->fd != -1) {
- /* FIXME: possibility to use .lock file instead */
- if (file_wait_lock(mcf->fd, type) <= 0) {
- index_cf_set_syscall_error(mcf, "file_wait_lock()");
- return FALSE;
- }
- }
-
- mcf->lock_type = type;
-
- if (type != F_UNLCK && !mcf->syncing) {
- mcf->syncing = TRUE;
- if (!custom_flags_check_sync(mcf)) {
- mcf->syncing = FALSE;
- return FALSE;
- }
-
- /* syncing may have changed locking, do it again */
- if (!lock_file(mcf, type)) {
- mcf->syncing = FALSE;
- return FALSE;
- }
-
- mcf->syncing = FALSE;
- }
- return TRUE;
-}
-
-int mail_custom_flags_open_or_create(struct mail_index *index)
-{
- struct mail_custom_flags *mcf;
- const char *path;
- int fd, readonly;
-
- readonly = index->mailbox_readonly;
-
- if (index->control_dir != NULL) {
- path = t_strconcat(index->control_dir, "/",
- CUSTOM_FLAGS_FILE_NAME, NULL);
- fd = !readonly ? open(path, O_RDWR | O_CREAT, 0660) :
- open(path, O_RDONLY);
- if (fd == -1 && errno == EACCES) {
- fd = open(path, O_RDONLY);
- readonly = TRUE;
- }
- if (fd == -1 && errno != EACCES && errno != ENOENT &&
- !ENOSPACE(errno)) {
- index_file_set_syscall_error(index, path, "open()");
- return FALSE;
- }
- } else {
- path = NULL;
- fd = -1;
- }
-
- mcf = i_new(struct mail_custom_flags, 1);
- mcf->index = index;
- mcf->filepath = fd != -1 ? i_strdup(path) :
- i_strdup_printf("(in-memory custom flags for %s)",
- index->mailbox_path);
- mcf->fd = fd;
- mcf->noupdate = mcf->fd == -1 || readonly;
-
- if (fd != -1) {
- if (!update_mmap(mcf)) {
- (void)close(mcf->fd);
- mcf->fd = -1;
- mcf->noupdate = TRUE;
- }
-
- if (mcf->mmap_length < HEADER_SIZE && !mcf->noupdate) {
- /* we just created it, write the header */
- mcf->syncing = TRUE;
- if (!custom_flags_init(mcf) || !update_mmap(mcf)) {
- (void)close(mcf->fd);
- mcf->fd = -1;
- mcf->noupdate = TRUE;
- }
- mcf->syncing = FALSE;
- }
- }
-
- mcf->index->allow_new_custom_flags = mcf->fd != -1;
-
- custom_flags_sync(mcf);
-
- index->custom_flags = mcf;
- return TRUE;
-}
-
-void mail_custom_flags_free(struct mail_custom_flags *mcf)
-{
- int i;
-
- for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++)
- i_free(mcf->custom_flags[i]);
-
- if (mcf->mmap_base != NULL) {
- if (munmap(mcf->mmap_base, mcf->mmap_length) < 0)
- index_cf_set_syscall_error(mcf, "munmap()");
- }
-
- if (mcf->fd != -1) {
- if (close(mcf->fd) < 0)
- index_cf_set_syscall_error(mcf, "close()");
- }
-
- i_free(mcf->filepath);
- i_free(mcf);
-}
-
-static int custom_flags_update_counter(struct mail_custom_flags *mcf)
-{
- int i;
-
- if (lseek(mcf->fd, 0, SEEK_SET) < 0)
- return index_cf_set_syscall_error(mcf, "lseek()");
-
- for (i = COUNTER_SIZE-1; i >= 0; i--) {
- if (mcf->sync_counter[i] == '9') {
- mcf->sync_counter[i] = 'A';
- break;
- }
-
- if (mcf->sync_counter[i] == 'F') {
- /* digit wrapped, update next one */
- mcf->sync_counter[i] = '0';
- } else {
- mcf->sync_counter[i]++;
- break;
- }
- }
-
- if (write_full(mcf->fd, mcf->sync_counter, COUNTER_SIZE) < 0)
- return index_cf_set_syscall_error(mcf, "write_full()");
-
- mcf->changed = TRUE;
- return TRUE;
-}
-
-static int custom_flags_add(struct mail_custom_flags *mcf,
- int idx, const char *name)
-{
- const char *buf;
- size_t len;
- off_t pos;
-
- i_assert(idx < MAIL_CUSTOM_FLAGS_COUNT);
-
- /* first update the sync counter */
- if (!custom_flags_update_counter(mcf))
- return FALSE;
-
- /* add the flag */
- pos = lseek(mcf->fd, 0, SEEK_END);
- if (pos < 0)
- return index_cf_set_syscall_error(mcf, "lseek()");
-
- if (pos != (off_t)mcf->mmap_length) {
- index_set_error(mcf->index, "Custom flags file %s was "
- "changed by someone while we were"
- "trying to modify it", mcf->filepath);
- return FALSE;
- }
-
- buf = t_strdup_printf("\n%d %s\n", idx, name);
- len = strlen(buf);
-
- if (((char *) mcf->mmap_base)[mcf->mmap_length-1] == '\n') {
- /* don't add the \n prefix */
- buf++;
- len--;
- }
-
- if (write_full(mcf->fd, buf, len) < 0)
- return index_cf_set_syscall_error(mcf, "write_full()");
-
- if (!update_mmap(mcf))
- return FALSE;
-
- return TRUE;
-}
-
-static int custom_flags_remove(struct mail_custom_flags *mcf, unsigned int idx)
-{
- char *data, *data_end, *line;
- unsigned int num;
- int pos, linelen;
-
- data = mcf->mmap_base;
- data_end = data + mcf->mmap_length;
-
- while (data != data_end) {
- if (*data != '\n') {
- data++;
- continue;
- }
-
- /* beginning of line, get the index */
- if (data+1 == data_end)
- break;
- line = ++data;
-
- num = 0;
- while (data != data_end && *data >= '0' && *data <= '9') {
- num = num*10 + (*data-'0');
- data++;
- }
-
- if (num == idx) {
- /* remove this line */
- while (data != data_end && data[-1] != '\n')
- data++;
-
- linelen = (int) (data - line);
- pos = (int) (data - (char *) mcf->mmap_base);
- memmove(line, data, mcf->mmap_length - pos);
-
- mcf->mmap_length -= linelen;
- if (ftruncate(mcf->fd, (off_t) mcf->mmap_length) < 0) {
- index_cf_set_syscall_error(mcf, "ftruncate()");
- return FALSE;
- }
-
- return TRUE;
- }
- }
-
- return FALSE;
-}
-
-static int find_first_unused_flag(struct mail_custom_flags *mcf)
-{
- int i;
-
- for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
- if (mcf->custom_flags[i] == NULL)
- return i;
- }
-
- return -1;
-}
-
-static void remove_unused_custom_flags(struct mail_custom_flags *mcf,
- enum mail_flags used_flags)
-{
- unsigned int i;
-
- for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
- if ((used_flags & (1 << (i + MAIL_CUSTOM_FLAG_1_BIT))) == 0) {
- i_free(mcf->custom_flags[i]);
- mcf->custom_flags[i] = NULL;
-
- custom_flags_remove(mcf, i);
- }
- }
-}
-
-static enum mail_flags get_used_flags(struct mail_custom_flags *mcf)
-{
- struct mail_index_record *rec;
- enum mail_flags used_flags;
-
- used_flags = 0;
-
- rec = mcf->index->lookup(mcf->index, 1);
- while (rec != NULL) {
- used_flags |= rec->msg_flags;
- rec = mcf->index->next(mcf->index, rec);
- }
-
- return used_flags;
-}
-
-static int get_flag_index(struct mail_custom_flags *mcf, const char *flag,
- int index_hint)
-{
- int i, first_empty;
-
- if (index_hint >= 0 && index_hint < MAIL_CUSTOM_FLAGS_COUNT) {
- if (mcf->custom_flags[index_hint] != NULL &&
- strcasecmp(mcf->custom_flags[index_hint], flag) == 0)
- return index_hint;
- }
-
- /* check existing flags */
- for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
- if (mcf->custom_flags[i] != NULL) {
- i_assert(mcf->custom_flags[i] != '\0');
- if (strcasecmp(mcf->custom_flags[i], flag) == 0)
- return i;
- }
- }
-
- if (mcf->noupdate)
- return -1;
-
- if (mcf->lock_type != F_WRLCK) {
- /* unlock + write lock, don't directly change from
- read -> write lock to prevent deadlocking */
- if (!lock_file(mcf, F_UNLCK) || !lock_file(mcf, F_WRLCK))
- return -1;
-
- /* list may have already changed between the lock changes,
- check again */
- return get_flag_index(mcf, flag, -1);
- }
-
- /* new flag, add it. first find the first free flag, note that
- unlock+lock might have just changed it. */
- first_empty = find_first_unused_flag(mcf);
- if (first_empty == -1) {
- /* all custom flags are used, see if some of them are unused */
- remove_unused_custom_flags(mcf, get_used_flags(mcf));
-
- first_empty = find_first_unused_flag(mcf);
- if (first_empty == -1) {
- /* everything is in use */
- return -1;
- }
- }
-
- if (!custom_flags_add(mcf, first_empty, flag))
- return -1;
-
- mcf->index->set_flags |= MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS;
-
- mcf->custom_flags[first_empty] = i_strdup(flag);
- return first_empty;
-}
-
-int mail_custom_flags_fix_list(struct mail_custom_flags *mcf,
- enum mail_flags *flags,
- const char *custom_flags[], unsigned int count)
-{
- enum mail_flags oldflags, flag;
- int i, idx;
-
- i_assert(count < 32);
-
- if ((*flags & MAIL_CUSTOM_FLAGS_MASK) == 0)
- return 1;
-
- if (!lock_file(mcf, F_RDLCK))
- return -1;
-
- oldflags = *flags;
- *flags &= MAIL_SYSTEM_FLAGS_MASK;
-
- flag = MAIL_CUSTOM_FLAG_1;
- for (i = 0; i < (int)count; i++, flag <<= 1) {
- if ((oldflags & flag) && custom_flags[i] != NULL) {
- i_assert(*custom_flags[i] != '\0');
-
- idx = get_flag_index(mcf, custom_flags[i], i);
- if (idx == -1) {
- (void)lock_file(mcf, F_UNLCK);
- return 0;
- }
- *flags |= 1 << (idx + MAIL_CUSTOM_FLAG_1_BIT);
- }
- }
-
- if (!lock_file(mcf, F_UNLCK))
- return -1;
-
- return 1;
-}
-
-const char **mail_custom_flags_list_get(struct mail_custom_flags *mcf)
-{
- return (const char **) mcf->custom_flags;
-}
-
-int mail_custom_flags_has_changes(struct mail_custom_flags *mcf)
-{
- if (!mcf->changed)
- return FALSE;
- else {
- mcf->changed = FALSE;
- return TRUE;
- }
-}
+++ /dev/null
-#ifndef __MAIL_CUSTOM_FLAGS_H
-#define __MAIL_CUSTOM_FLAGS_H
-
-/* NOTE: Contains it's own locking, unrelated to index locks. */
-
-#include "mail-index.h"
-
-#define CUSTOM_FLAGS_FILE_NAME ".customflags"
-
-int mail_custom_flags_open_or_create(struct mail_index *index);
-void mail_custom_flags_free(struct mail_custom_flags *mcf);
-
-/* Change custom flags so that they reflect the real flag numbers in
- the file. Initially flags contains the custom flags in the order of the
- specified list, it's modified to reflect the actual list. Returns 1 if ok,
- 0 if number of custom flags exceeded or -1 if error */
-int mail_custom_flags_fix_list(struct mail_custom_flags *mcf,
- enum mail_flags *flags,
- const char *custom_flags[], unsigned int count);
-
-/* Returns a pointer to list of flags. Note that calls to
- mail_cutom_flags_fix_list() may modify the flags in the returned list.
- It can modify only the flags that aren't in use anywhere, so this should
- be safe. */
-const char **mail_custom_flags_list_get(struct mail_custom_flags *mcf);
-
-/* Returns TRUE if there's been any changes since this function was
- called last time, or since open if this is the first call. */
-int mail_custom_flags_has_changes(struct mail_custom_flags *mcf);
-
-#endif
+++ /dev/null
-/* Copyright (C) 2003 Timo Sirainen */
-
-#include "lib.h"
-#include "file-set-size.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-
-#include <unistd.h>
-
-struct mail_index_record *mail_index_next(struct mail_index *index,
- struct mail_index_record *rec)
-{
- i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
- i_assert(rec >= INDEX_RECORD_AT(index, 0));
-
- return rec+1 == INDEX_END_RECORD(index) ? NULL : rec+1;
-}
-
-static int compress(struct mail_index *index, unsigned int remove_first_idx,
- unsigned int remove_last_idx)
-{
- struct mail_index_record *rec = INDEX_RECORD_AT(index, 0);
- unsigned int idx_limit, count;
-
- idx_limit = MAIL_INDEX_RECORD_COUNT(index);
- count = remove_last_idx - remove_first_idx + 1;
-
- memmove(rec + remove_first_idx, rec + remove_last_idx + 1,
- (idx_limit - remove_last_idx - 1) * sizeof(*rec));
-
- index->header->used_file_size -= sizeof(*rec) * count;
- index->mmap_used_length -= sizeof(*rec) * count;
-
- /* not really needed since append() will initialize it as well,
- but may help preventing problems if change is only partially
- written to disk */
- memset((char *) rec + index->mmap_used_length, 0, sizeof(*rec) * count);
-
- return mail_index_truncate(index);
-}
-
-int mail_index_expunge_record_range(struct mail_index *index,
- struct mail_index_record *first_rec,
- struct mail_index_record *last_rec)
-{
- struct mail_index_record *rec;
- unsigned int first_idx, last_idx, idx_limit;
-
- i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
- first_idx = INDEX_RECORD_INDEX(index, first_rec);
- last_idx = INDEX_RECORD_INDEX(index, last_rec);
- idx_limit = MAIL_INDEX_RECORD_COUNT(index);
-
- i_assert(first_idx <= last_idx);
- i_assert(last_idx < idx_limit);
-
- index->header->messages_count -= last_idx - first_idx + 1;
- for (rec = first_rec; rec <= last_rec; rec++)
- mail_index_mark_flag_changes(index, rec, rec->msg_flags, 0);
-
- return compress(index, first_idx, last_idx);
-}
-
-struct mail_index_record *mail_index_lookup(struct mail_index *index,
- unsigned int seq)
-{
- i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
- i_assert(seq > 0);
-
- if (seq > index->header->messages_count)
- return NULL;
-
- return INDEX_RECORD_AT(index, seq-1);
-}
-
-struct mail_index_record *
-mail_index_lookup_uid_range(struct mail_index *index, unsigned int first_uid,
- unsigned int last_uid, unsigned int *seq_r)
-{
- struct mail_index_record *rec_p;
- unsigned int idx_limit, idx, left_idx, right_idx;
-
- i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
- i_assert(first_uid > 0);
- i_assert(first_uid <= last_uid);
-
- rec_p = INDEX_RECORD_AT(index, 0);
- idx_limit = MAIL_INDEX_RECORD_COUNT(index);
-
- idx = 0;
- left_idx = 0;
- right_idx = idx_limit;
-
- while (left_idx < right_idx) {
- idx = (left_idx + right_idx) / 2;
-
- if (rec_p[idx].uid < first_uid)
- left_idx = idx+1;
- else if (rec_p[idx].uid > first_uid)
- right_idx = idx;
- else
- break;
- }
-
- if (rec_p[idx].uid < first_uid || rec_p[idx].uid > last_uid) {
- /* could still be the next one */
- idx++;
- if (idx == idx_limit ||
- rec_p[idx].uid < first_uid || rec_p[idx].uid > last_uid) {
- if (seq_r != NULL) *seq_r = 0;
- return NULL;
- }
- }
-
- if (seq_r != NULL)
- *seq_r = idx + 1;
- return rec_p + idx;
-}
-
-int mail_index_compress(struct mail_index *index)
-{
- size_t diff;
- off_t new_file_size;
-
- if (index->header_size >= sizeof(struct mail_index_header))
- return TRUE;
-
- if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
-
- /* make sure the file is large enough */
- diff = sizeof(struct mail_index_header) - index->header_size;
- if (index->mmap_used_length + diff > index->mmap_full_length) {
- /* mmap_update ftruncates the file to multiples of
- mail_index_record, make sure we grow it enough here. */
- new_file_size = index->mmap_used_length + diff +
- (sizeof(struct mail_index_record) -
- (diff % sizeof(struct mail_index_record)));
- if (file_set_size(index->fd, new_file_size) < 0) {
- index_set_syscall_error(index, "file_set_size()");
- return FALSE;
- }
-
- index->header->master_sync_id++;
- if (!mail_index_mmap_update(index))
- return FALSE;
- }
-
- /* if we break, we'll have to rebuild it completely */
- index->header->flags |= MAIL_INDEX_HDR_FLAG_REBUILD;
- if (!mail_index_fmdatasync(index, index->header_size))
- return FALSE;
-
- memmove((char *) index->mmap_base + sizeof(struct mail_index_header),
- (char *) index->mmap_base + index->header_size,
- index->mmap_used_length - index->header_size);
- memset((char *) index->mmap_base + index->header_size, 0, diff);
-
- index->mmap_used_length += diff;
- index->header_size = sizeof(struct mail_index_header);
-
- index->header->header_size = sizeof(struct mail_index_header);
- index->header->used_file_size += diff;
- index->header->master_sync_id++;
-
- if (!mail_index_fmdatasync(index, index->mmap_used_length))
- return FALSE;
-
- index->header->flags &= ~MAIL_INDEX_HDR_FLAG_REBUILD;
- return mail_index_mmap_update(index);
-}
-
-int mail_index_truncate(struct mail_index *index)
-{
- uoff_t empty_space, truncate_threshold;
-
- i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
- if (index->mmap_full_length <= INDEX_FILE_MIN_SIZE(index) ||
- index->anon_mmap)
- return TRUE;
-
- /* really truncate the file only when it's almost empty */
- empty_space = index->mmap_full_length - index->mmap_used_length;
- truncate_threshold =
- index->mmap_full_length / 100 * INDEX_TRUNCATE_PERCENTAGE;
-
- if (empty_space > truncate_threshold) {
- index->mmap_full_length = index->mmap_used_length +
- (empty_space * INDEX_TRUNCATE_KEEP_PERCENTAGE / 100);
-
- /* keep the size record-aligned */
- index->mmap_full_length -= (index->mmap_full_length -
- index->header_size) %
- sizeof(struct mail_index_record);
-
- if (index->mmap_full_length < INDEX_FILE_MIN_SIZE(index))
- index->mmap_full_length = INDEX_FILE_MIN_SIZE(index);
-
- if (ftruncate(index->fd, (off_t)index->mmap_full_length) < 0)
- return index_set_syscall_error(index, "ftruncate()");
-
- index->header->master_sync_id++;
- }
-
- return TRUE;
-}
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-
-#define CHECK(field) \
- if (old_hdr->field != new_hdr->field) \
- i_warning("fsck %s: "#field" %u != %u", \
- index->filepath, old_hdr->field, new_hdr->field);
-
-
-static void print_differences(struct mail_index *index,
- struct mail_index_header *old_hdr,
- struct mail_index_header *new_hdr)
-{
- CHECK(next_uid);
-
- CHECK(messages_count);
- CHECK(seen_messages_count);
- CHECK(deleted_messages_count);
- CHECK(last_nonrecent_uid);
-
- if (old_hdr->first_unseen_uid_lowwater >
- new_hdr->first_unseen_uid_lowwater) {
- i_warning("fsck %s: first_unseen_uid_lowwater %u > %u",
- index->filepath,
- old_hdr->first_unseen_uid_lowwater,
- new_hdr->first_unseen_uid_lowwater);
- }
-
- if (old_hdr->first_deleted_uid_lowwater >
- new_hdr->first_deleted_uid_lowwater) {
- i_warning("fsck %s: first_deleted_uid_lowwater %u > %u",
- index->filepath,
- old_hdr->first_deleted_uid_lowwater,
- new_hdr->first_deleted_uid_lowwater);
- }
-}
-
-int mail_index_fsck(struct mail_index *index)
-{
- struct mail_index_header old_hdr, *hdr;
- struct mail_index_record *rec, *end_rec;
- unsigned int max_uid;
-
- i_assert(index->lock_type != MAIL_LOCK_SHARED);
-
- if (!mail_index_compress(index))
- return FALSE;
-
- /* then we verify only the fields in the header. other problems will
- be noticed and fixed while reading the messages. */
- hdr = index->header;
- memcpy(&old_hdr, hdr, sizeof(struct mail_index_header));
-
- hdr->messages_count = 0;
- hdr->seen_messages_count = 0;
- hdr->deleted_messages_count = 0;
-
- hdr->first_unseen_uid_lowwater = 0;
- hdr->first_deleted_uid_lowwater = 0;
-
- rec = INDEX_RECORD_AT(index, 0);
- end_rec = INDEX_END_RECORD(index);
-
- max_uid = 0;
- for (; rec < end_rec; rec++) {
- if (rec->uid < max_uid) {
- index_set_corrupted(index, "UIDs are not ordered "
- "(%u < %u)", rec->uid, max_uid);
- return FALSE;
- }
- max_uid = rec->uid;
-
- if (rec->msg_flags & MAIL_SEEN)
- hdr->seen_messages_count++;
- else if (hdr->first_unseen_uid_lowwater == 0)
- hdr->first_unseen_uid_lowwater = rec->uid;
-
- if (rec->msg_flags & MAIL_DELETED) {
- if (hdr->first_deleted_uid_lowwater == 0)
- hdr->first_deleted_uid_lowwater = rec->uid;
- hdr->deleted_messages_count++;
- }
- hdr->messages_count++;
- }
-
- if (hdr->next_uid <= max_uid)
- hdr->next_uid = max_uid+1;
- if (hdr->last_nonrecent_uid >= hdr->next_uid)
- hdr->last_nonrecent_uid = hdr->next_uid-1;
-
- if (hdr->first_unseen_uid_lowwater == 0)
- hdr->first_unseen_uid_lowwater = hdr->next_uid;
- if (hdr->first_deleted_uid_lowwater == 0)
- hdr->first_deleted_uid_lowwater = hdr->next_uid;
-
- print_differences(index, &old_hdr, hdr);
-
- /* FSCK flag is removed automatically by set_lock() */
- return TRUE;
-}
+++ /dev/null
-/* Copyright (C) 2002-2003 Timo Sirainen */
-
-#include "lib.h"
-#include "ioloop.h"
-#include "file-lock.h"
-#include "file-set-size.h"
-#include "hostpid.h"
-#include "mmap-util.h"
-#include "unlink-lockfiles.h"
-#include "write-full.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-#include "mail-modifylog.h"
-#include "mail-custom-flags.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-
-static int mail_index_open_init(struct mail_index *index,
- enum mail_index_open_flags flags)
-{
- struct mail_index_header *hdr;
-
- hdr = index->header;
-
- index->maildir_have_new =
- (hdr->flags & MAIL_INDEX_FLAG_MAILDIR_NEW) != 0;
-
- if ((hdr->flags & MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES) != 0)
- index->next_dirty_flags_flush = ioloop_time;
-
- /* update \Recent message counters */
- if ((flags & MAIL_INDEX_OPEN_FLAG_UPDATE_RECENT) != 0 &&
- hdr->last_nonrecent_uid != hdr->next_uid-1) {
- /* keep last_recent_uid to next_uid-1 */
- if (index->lock_type == MAIL_LOCK_SHARED) {
- if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
- return FALSE;
- }
-
- if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
-
- index->first_recent_uid = index->header->last_nonrecent_uid+1;
- index->header->last_nonrecent_uid = index->header->next_uid-1;
- } else {
- index->first_recent_uid = hdr->last_nonrecent_uid+1;
- }
-
- if (hdr->next_uid >= MAX_ALLOWED_UID - 1000) {
- /* UID values are getting too high, rebuild index */
- index->set_flags |= MAIL_INDEX_HDR_FLAG_REBUILD;
- }
-
- if (index->lock_type == MAIL_LOCK_EXCLUSIVE) {
- /* finally reset the modify log marks, fsck or syncing might
- have deleted some messages, and since we're only just
- opening the index, there's no need to remember them */
- if (!mail_modifylog_mark_synced(index->modifylog))
- return FALSE;
- }
-
- return TRUE;
-}
-
-static int index_open_and_fix(struct mail_index *index,
- enum mail_index_open_flags flags)
-{
- int rebuilt;
-
- if (index->header_size < sizeof(struct mail_index_header)) {
- /* upgrading from older index file. */
- if (!mail_index_compress(index))
- return FALSE;
- }
-
- if (!mail_cache_open_or_create(index))
- return FALSE;
-
- /* custom flags file needs to be open before rebuilding index */
- if (!mail_custom_flags_open_or_create(index))
- return FALSE;
-
- if ((index->header->flags & MAIL_INDEX_HDR_FLAG_REBUILD) != 0 ||
- (index->set_flags & MAIL_INDEX_HDR_FLAG_REBUILD) != 0) {
-
- if (!index->rebuild(index))
- return FALSE;
-
- if ((index->header->flags & MAIL_INDEX_HDR_FLAG_REBUILD) != 0)
- return FALSE;
-
- /* no inconsistency problems since we're still opening
- the index */
- index->inconsistent = FALSE;
- rebuilt = TRUE;
- } else {
- rebuilt = FALSE;
- }
-
- if ((flags & _MAIL_INDEX_OPEN_FLAG_CREATING) == 0) {
- if (!mail_modifylog_open_or_create(index))
- return FALSE;
- } else {
- if (!mail_modifylog_create(index))
- return FALSE;
- }
-
- if (index->header->flags & MAIL_INDEX_HDR_FLAG_FSCK) {
- /* index needs fscking */
- if (!index->fsck(index))
- return FALSE;
- }
-
- if (!rebuilt) {
- /* sync ourself. do it before compression which may happen
- as a result of this. */
- if (!index->sync_and_lock(index, FALSE,
- MAIL_LOCK_SHARED, NULL) &&
- !index->nodiskspace)
- return FALSE;
- }
-
- /* we never want to keep shared lock if syncing happens to set it.
- either exclusive or nothing (NOTE: drop it directly, not through
- index->set_lock() so mbox lock won't be affected). */
- if (index->lock_type == MAIL_LOCK_SHARED) {
- if (!mail_index_set_lock(index, MAIL_LOCK_UNLOCK))
- return FALSE;
- }
-
- if ((flags & MAIL_INDEX_OPEN_FLAG_FAST) == 0) {
- if (index->header->flags & MAIL_INDEX_HDR_FLAG_COMPRESS) {
- /* remove deleted blocks from index file */
- if (!mail_index_compress(index))
- return FALSE;
- }
-
- if (index->header->flags & MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE) {
- /* remove unused space from index data file. */
- if (!mail_cache_compress(index->cache))
- return FALSE;
- }
- }
-
- if (!mail_index_open_init(index, flags))
- return FALSE;
-
- return TRUE;
-}
-
-static int mail_index_read_header(struct mail_index *index,
- struct mail_index_header *hdr)
-{
- ssize_t ret;
-
- if (lseek(index->fd, 0, SEEK_SET) < 0) {
- index_set_syscall_error(index, "seek()");
- return -1;
- }
-
- ret = read(index->fd, hdr, sizeof(*hdr));
- if (ret < 0) {
- index_set_syscall_error(index, "read()");
- return -1;
- }
-
- if (ret != sizeof(*hdr)) {
- /* missing data */
- return 0;
- }
-
- return 1;
-}
-
-static int mail_index_init_file(struct mail_index *index,
- const struct mail_index_header *hdr)
-{
- uoff_t file_size;
-
- if (lseek(index->fd, 0, SEEK_SET) < 0) {
- index_set_syscall_error(index, "lseek()");
- return FALSE;
- }
-
- if (write_full(index->fd, hdr, sizeof(*hdr)) < 0) {
- index_set_syscall_error(index, "write_full()");
- return FALSE;
- }
-
- file_size = sizeof(*hdr) +
- INDEX_MIN_RECORDS_COUNT * sizeof(struct mail_index_record);
- if (file_set_size(index->fd, (off_t)file_size) < 0) {
- index_set_syscall_error(index, "file_set_size()");
- return FALSE;
- }
-
- return TRUE;
-}
-
-static void get_compat_data(unsigned char compat_data[4])
-{
-#ifndef WORDS_BIGENDIAN
- compat_data[0] = MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
-#else
- compat_data[0] = 0;
-#endif
- compat_data[1] = sizeof(uoff_t);
- compat_data[2] = sizeof(time_t);
- compat_data[3] = 0;
-}
-
-void mail_index_init_header(struct mail_index_header *hdr)
-{
- i_assert(sizeof(struct mail_index_header) < 256);
-
- memset(hdr, 0, sizeof(*hdr));
- hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
- hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
- hdr->header_size = (uint8_t)sizeof(struct mail_index_header);
- get_compat_data(hdr->compat_data);
-
- hdr->indexid = ioloop_time;
-
- /* mark the index requiring rebuild - rebuild() removes this flag
- when it succeeds */
- hdr->flags = MAIL_INDEX_HDR_FLAG_REBUILD;
-
- hdr->used_file_size = sizeof(struct mail_index_header);
- hdr->uid_validity = ioloop_time;
- hdr->next_uid = 1;
-}
-
-static void mail_index_cleanup_temp_files(const char *dir)
-{
- unlink_lockfiles(dir, t_strconcat(".temp.", my_hostname, ".", NULL),
- ".temp.", time(NULL) - TEMP_FILE_TIMEOUT);
-}
-
-void mail_index_init(struct mail_index *index, const char *dir)
-{
- size_t len;
-
- index->fd = -1;
-
- if (dir != NULL) {
- index->dir = i_strdup(dir);
-
- len = strlen(index->dir);
- if (index->dir[len-1] == '/')
- index->dir[len-1] = '\0';
- }
-
- index->mail_read_mmaped = getenv("MAIL_READ_MMAPED") != NULL;
-}
-
-static int mail_index_create_memory(struct mail_index *index,
- enum mail_index_open_flags flags)
-{
- if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
- return FALSE;
-
- flags |= _MAIL_INDEX_OPEN_FLAG_CREATING;
-
- index->header_size = sizeof(struct mail_index_header);
- index->mmap_full_length = INDEX_FILE_MIN_SIZE(index);
- index->mmap_base = mmap_anon(index->mmap_full_length);
- if (index->mmap_base == MAP_FAILED) {
- index->mmap_base = NULL;
- return index_set_error(index, "mmap_anon() failed: %m");
- }
-
- mail_index_init_header(index->mmap_base);
- index->header = index->mmap_base;
- index->mmap_used_length = index->header->used_file_size;
-
- index->anon_mmap = TRUE;
- index->lock_type = MAIL_LOCK_EXCLUSIVE;
- index->indexid = index->header->indexid;
- index->filepath = i_strdup_printf("(in-memory index for %s)",
- index->mailbox_path);
-
- if (!index_open_and_fix(index, flags)) {
- mail_index_close(index);
- return FALSE;
- }
-
- index->opened = TRUE;
- return TRUE;
-}
-
-static int mail_index_open_index(struct mail_index *index,
- enum mail_index_open_flags flags)
-{
- struct mail_index_header hdr;
- unsigned char compat_data[4];
- int ret;
-
- if ((flags & _MAIL_INDEX_OPEN_FLAG_CREATING) == 0)
- index->lock_type = MAIL_LOCK_SHARED;
- else
- index->lock_type = MAIL_LOCK_EXCLUSIVE;
-
- /* if index is being created, we'll wait here until it's finished */
- if (!mail_index_wait_lock(index, MAIL_LOCK_TO_FLOCK(index->lock_type)))
- return FALSE;
-#ifdef DEBUG
- if (index->mmap_base != NULL) {
- mprotect(index->mmap_base, index->mmap_used_length,
- PROT_READ|PROT_WRITE);
- }
-#endif
-
- if ((ret = mail_index_read_header(index, &hdr)) < 0)
- return FALSE;
- index->indexid = hdr.indexid;
-
- get_compat_data(compat_data);
- if (ret == 0 || hdr.major_version != MAIL_INDEX_MAJOR_VERSION ||
- (hdr.flags & MAIL_INDEX_HDR_FLAG_REBUILD) != 0 ||
- memcmp(compat_data, hdr.compat_data, sizeof(compat_data)) != 0 ||
- !mail_index_mmap_update(index)) {
- if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
- return FALSE;
-
- flags |= _MAIL_INDEX_OPEN_FLAG_CREATING;
-
- /* so, we're creating the index */
- if (index->lock_type != MAIL_LOCK_EXCLUSIVE) {
- /* have to get exclusive lock first */
- if (!mail_index_wait_lock(index, F_UNLCK))
- return FALSE;
- return mail_index_open_index(index, flags);
- }
-
- mail_index_init_header(&hdr);
- if (!mail_index_init_file(index, &hdr))
- return FALSE;
-
- if (!mail_index_mmap_update(index))
- return FALSE;
- }
-
- if (index->lock_type == MAIL_LOCK_SHARED) {
- /* we don't want to keep the shared lock while opening
- indexes. opening should work unlocked and some
- things want exclusive lock */
- if (!mail_index_wait_lock(index, F_UNLCK))
- return FALSE;
- index->lock_type = MAIL_LOCK_UNLOCK;
- }
-
- if (!index_open_and_fix(index, flags))
- return FALSE;
-
- if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
- return FALSE;
-
- index->opened = TRUE;
- return TRUE;
-}
-
-int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags)
-{
- const char *path;
-
- i_assert(!index->opened);
-
- if (index->dir == NULL)
- return mail_index_create_memory(index, flags);
-
- mail_index_cleanup_temp_files(index->dir);
-
- /* open/create the file */
- path = t_strconcat(index->dir, "/", INDEX_FILE_PREFIX, NULL);
- if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) != 0)
- index->fd = open(path, O_RDWR | O_CREAT, 0660);
- else
- index->fd = open(path, O_RDWR);
- if (index->fd == -1) {
- if (errno != ENOENT)
- index_file_set_syscall_error(index, path, "open()");
- return mail_index_create_memory(index, flags);
- }
-
- index->filepath = i_strdup(path);
-
- if (!mail_index_open_index(index, flags)) {
- mail_index_close(index);
- return mail_index_create_memory(index, flags);
- }
-
- return TRUE;
-}
+++ /dev/null
-/* Copyright (C) 2002-2003 Timo Sirainen */
-
-#include "lib.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-
-#include <sys/mman.h>
-
-int mail_index_rebuild(struct mail_index *index)
-{
- if (!mail_index_set_lock(index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
-
- index->set_flags &= ~MAIL_INDEX_HDR_FLAG_REBUILD;
-
- /* reset the header */
- mail_index_init_header(index->header);
- index->mmap_used_length = index->header->used_file_size;
-
- /* update indexid, which also means that our state has completely
- changed */
- index->indexid = index->header->indexid;
- index->inconsistent = TRUE;
- index->rebuilding = TRUE;
-
- if (!index->anon_mmap) {
- if (msync(index->mmap_base, index->header_size, MS_SYNC) < 0)
- return index_set_syscall_error(index, "msync()");
- }
-
- if (!mail_cache_truncate(index->cache))
- return FALSE;
-
- /* read the mails by syncing */
- if (!index->sync_and_lock(index, FALSE, MAIL_LOCK_UNLOCK, NULL))
- return FALSE;
-
- /* rebuild is complete - remove the flag */
- index->header->flags &= ~(MAIL_INDEX_HDR_FLAG_REBUILD |
- MAIL_INDEX_HDR_FLAG_FSCK);
- index->header->flags |= index->set_flags;
- index->set_flags = 0;
-
- index->rebuilding = FALSE;
- return TRUE;
-}
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "istream.h"
-#include "hostpid.h"
-#include "file-lock.h"
-#include "message-size.h"
-#include "message-part-serialize.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-
-#include <unistd.h>
-#include <fcntl.h>
-
-int index_set_error(struct mail_index *index, const char *fmt, ...)
-{
- va_list va;
-
- i_free(index->error);
-
- if (fmt == NULL)
- index->error = NULL;
- else {
- va_start(va, fmt);
- index->error = i_strdup_vprintf(fmt, va);
- va_end(va);
-
- i_error("%s", index->error);
- }
-
- return FALSE;
-}
-
-int index_set_corrupted(struct mail_index *index, const char *fmt, ...)
-{
- va_list va;
-
- INDEX_MARK_CORRUPTED(index);
- index->inconsistent = TRUE;
-
- va_start(va, fmt);
- t_push();
- index_set_error(index, "Corrupted index file %s: %s",
- index->filepath, t_strdup_vprintf(fmt, va));
- t_pop();
- va_end(va);
-
- return FALSE;
-}
-
-int index_set_syscall_error(struct mail_index *index, const char *function)
-{
- i_assert(function != NULL);
-
- if (ENOSPACE(errno)) {
- index->nodiskspace = TRUE;
- return FALSE;
- }
-
- index_set_error(index, "%s failed with index file %s: %m",
- function, index->filepath);
- return FALSE;
-}
-
-int index_file_set_syscall_error(struct mail_index *index, const char *filepath,
- const char *function)
-{
- i_assert(filepath != NULL);
- i_assert(function != NULL);
-
- if (ENOSPACE(errno)) {
- index->nodiskspace = TRUE;
- return FALSE;
- }
-
- index_set_error(index, "%s failed with file %s: %m",
- function, filepath);
-
- return FALSE;
-}
-
-void index_reset_error(struct mail_index *index)
-{
- if (index->error != NULL) {
- i_free(index->error);
- index->error = NULL;
- }
-
- index->nodiskspace = FALSE;
-}
-
-int mail_index_create_temp_file(struct mail_index *index, const char **path)
-{
- int fd;
-
- /* use ".temp.host.pid" as temporary file name. unlink() it first,
- just to be sure it's not symlinked somewhere for some reason..
- FIXME: this function should rather be removed entirely. With
- in-memory indexes index->dir is NULL, so we fallback to /tmp
- so that mbox rewriting doesn't crash. */
- *path = t_strconcat(index->dir != NULL ? index->dir : "/tmp",
- "/.temp.", my_hostname, ".", my_pid, NULL);
- (void)unlink(*path);
-
- /* usage of O_EXCL isn't exactly needed since the path should be
- trusted, but it shouldn't hurt either - if creating file fails
- because of it, it's because something must be wrong (race
- condition). also, might not won't work through NFS but that
- can't be helped. */
- fd = open(*path, O_RDWR | O_CREAT | O_EXCL, 0660);
- if (fd == -1) {
- if (ENOSPACE(errno))
- index->nodiskspace = TRUE;
- else {
- index_set_error(index, "Can't create temp index %s: %m",
- *path);
- }
- }
-
- return fd;
-}
-
-static void mail_index_lock_notify(unsigned int secs_left, void *context)
-{
- struct mail_index *index = context;
-
- if (index->lock_notify_cb == NULL)
- return;
-
- index->lock_notify_cb(MAIL_LOCK_NOTIFY_INDEX_ABORT, secs_left,
- index->lock_notify_context);
-}
-
-int mail_index_wait_lock(struct mail_index *index, int lock_type)
-{
- int ret;
-
- ret = file_wait_lock_full(index->fd, lock_type, DEFAULT_LOCK_TIMEOUT,
- mail_index_lock_notify, index);
- if (ret < 0)
- return index_set_syscall_error(index, "file_wait_lock()");
-
- if (ret == 0) {
- index_set_error(index, "Timeout while waiting for release of "
- "%s fcntl() lock for index file %s",
- lock_type == F_WRLCK ? "exclusive" : "shared",
- index->filepath);
- index->index_lock_timeout = TRUE;
- return FALSE;
- }
-
- return TRUE;
-
-}
+++ /dev/null
-#ifndef __MAIL_INDEX_UTIL_H
-#define __MAIL_INDEX_UTIL_H
-
-/* Get index's lock state as mprotect() argument */
-#define MAIL_INDEX_PROT(index) \
- ((index)->lock_type == MAIL_LOCK_EXCLUSIVE ? (PROT_READ|PROT_WRITE) : \
- (index)->lock_type == MAIL_LOCK_SHARED || !(index)->opened ? \
- PROT_READ : PROT_NONE)
-
-/* DEBUG: Force mmap() locks with mprotect() */
-#ifdef DEBUG
-# define debug_mprotect(mmap_base, mmap_length, index) \
- mprotect(mmap_base, mmap_length, MAIL_INDEX_PROT(index))
-#else
-# define debug_mprotect(mmap_base, mmap_length, index)
-#endif
-
-/* Set the current error message */
-int index_set_error(struct mail_index *index, const char *fmt, ...)
- __attr_format__(2, 3);
-
-/* "Error in index file %s: ...". Also marks the index file as corrupted. */
-int index_set_corrupted(struct mail_index *index, const char *fmt, ...)
- __attr_format__(2, 3);
-
-/* "%s failed with index file %s: %m" */
-int index_set_syscall_error(struct mail_index *index, const char *function);
-
-/* "%s failed with file %s: %m" */
-int index_file_set_syscall_error(struct mail_index *index, const char *filepath,
- const char *function);
-
-/* Reset the current error */
-void index_reset_error(struct mail_index *index);
-
-/* Create temporary file into index's directory. Returns opened file handle
- and sets *path to the full path of the created file. */
-int mail_index_create_temp_file(struct mail_index *index, const char **path);
-
-/* Wrapper to file_set_lock(), also calling index's lock notify callback. */
-int mail_index_wait_lock(struct mail_index *index, int lock_type);
-
-#endif
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "ioloop.h"
-#include "file-lock.h"
-#include "file-set-size.h"
-#include "mmap-util.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-#include "mail-modifylog.h"
-#include "mail-custom-flags.h"
-
-#include <unistd.h>
-#include <fcntl.h>
-
-static void update_header(struct mail_index *index)
-{
- struct mail_index_header *hdr = index->header;
-
- index->cache_sync_id = hdr->cache_sync_id;
- index->log_sync_id = hdr->log_sync_id;
- index->sync_stamp = hdr->sync_stamp;
- index->sync_size = hdr->sync_size;
-}
-
-static int mmap_verify(struct mail_index *index)
-{
- struct mail_index_header *hdr;
- unsigned int extra;
-
- index->mmap_used_length = 0;
-
- if (index->mmap_full_length < sizeof(struct mail_index_header)) {
- index_set_corrupted(index, "File too small");
- return FALSE;
- }
-
- /* keep the header set even if we fail, so we can update the flags */
- hdr = index->mmap_base;
- index->header = hdr;
- index->header_size = hdr->header_size;
-
- if (index->header_size > index->mmap_full_length) {
- index_set_corrupted(index, "Invalid header_size in header "
- "(%"PRIuSIZE_T")", index->header_size);
- return FALSE;
- }
-
- extra = (index->mmap_full_length - index->header_size) %
- sizeof(struct mail_index_record);
-
- if (extra != 0) {
- /* partial write or corrupted -
- truncate the file to valid length */
- i_assert(!index->anon_mmap);
-
- index->mmap_full_length -= extra;
- (void)ftruncate(index->fd, (off_t)index->mmap_full_length);
- }
-
- if (hdr->used_file_size > index->mmap_full_length) {
- index_set_corrupted(index,
- "used_file_size larger than real file size "
- "(%u vs %"PRIuSIZE_T")",
- hdr->used_file_size,
- index->mmap_full_length);
- return FALSE;
- }
-
- if (hdr->used_file_size < index->header_size ||
- (hdr->used_file_size - index->header_size) %
- sizeof(struct mail_index_record) != 0) {
- index_set_corrupted(index, "Invalid used_file_size in header "
- "(%u)", hdr->used_file_size);
- return FALSE;
- }
-
- if (hdr->messages_count < hdr->seen_messages_count) {
- index_set_corrupted(index, "Invalid seen messages count "
- "(%u < %u)", hdr->messages_count,
- hdr->seen_messages_count);
- return FALSE;
- }
-
- if (hdr->messages_count < hdr->deleted_messages_count) {
- index_set_corrupted(index, "Invalid deleted messages count "
- "(%u < %u)", hdr->messages_count,
- hdr->deleted_messages_count);
- return FALSE;
- }
-
- index->master_sync_id = hdr->master_sync_id;
- index->mmap_used_length = hdr->used_file_size;
- update_header(index);
- return TRUE;
-}
-
-int mail_index_mmap_update(struct mail_index *index)
-{
- if (index->anon_mmap)
- return mmap_verify(index);
-
- if (index->mmap_base != NULL) {
- index->header = (struct mail_index_header *) index->mmap_base;
- update_header(index);
-
- if (index->mmap_invalidate) {
- if (msync(index->mmap_base,
- index->mmap_used_length,
- MS_SYNC | MS_INVALIDATE) < 0) {
- index_set_syscall_error(index, "msync()");
- return FALSE;
- }
- }
-
- /* make sure file size hasn't changed */
- if (index->header->master_sync_id == index->master_sync_id) {
- index->mmap_used_length = index->header->used_file_size;
- if (index->mmap_used_length > index->mmap_full_length) {
- i_panic("Index file size was grown without "
- "updating sync_id");
- }
- return TRUE;
- }
-
- if (!index->mmap_invalidate) {
- if (msync(index->mmap_base,
- index->mmap_used_length, MS_SYNC) < 0) {
- index_set_syscall_error(index, "msync()");
- return FALSE;
- }
- }
-
- if (munmap(index->mmap_base, index->mmap_full_length) < 0)
- return index_set_syscall_error(index, "munmap()");
- }
-
- index->mmap_base = mmap_rw_file(index->fd, &index->mmap_full_length);
- if (index->mmap_base == MAP_FAILED) {
- index->mmap_base = NULL;
- index->mmap_used_length = 0;
- index_set_syscall_error(index, "mmap()");
- return FALSE;
- }
-
- return mmap_verify(index);
-}
-
-void mail_index_close(struct mail_index *index)
-{
- if (index->set_flags != 0) {
- if (index->header != NULL) {
-#ifdef DEBUG
- mprotect(index->mmap_base, index->mmap_used_length,
- PROT_READ|PROT_WRITE);
-#endif
- index->header->flags |= index->set_flags;
- (void)msync(index->mmap_base, index->header_size,
- MS_SYNC);
- }
- index->set_flags = 0;
- }
-
- index->opened = FALSE;
- index->inconsistent = FALSE;
-
- index->lock_type = MAIL_LOCK_UNLOCK;
- index->header = NULL;
-
- if (index->fd != -1) {
- if (close(index->fd) < 0)
- index_set_syscall_error(index, "close()");
- index->fd = -1;
- }
-
- if (index->filepath != NULL) {
- i_free(index->filepath);
- index->filepath = NULL;
- }
-
- if (index->anon_mmap) {
- if (munmap_anon(index->mmap_base, index->mmap_full_length) < 0)
- index_set_syscall_error(index, "munmap_anon()");
- index->anon_mmap = FALSE;
- } else if (index->mmap_base != NULL) {
- if (munmap(index->mmap_base, index->mmap_full_length) < 0)
- index_set_syscall_error(index, "munmap()");
- }
- index->mmap_base = NULL;
-
- if (index->cache != NULL) {
- mail_cache_free(index->cache);
- index->cache = NULL;
- }
-
- if (index->modifylog != NULL) {
- mail_modifylog_free(index->modifylog);
- index->modifylog = NULL;
- }
-
- if (index->custom_flags != NULL) {
- mail_custom_flags_free(index->custom_flags);
- index->custom_flags = NULL;
- }
-
- if (index->error != NULL) {
- i_free(index->error);
- index->error = NULL;
- }
-}
-
-static int mail_index_sync_file(struct mail_index *index)
-{
- unsigned int i;
- int failed, fsync_fds[3];
-
- if (index->anon_mmap)
- return TRUE;
-
- for (i = 0; i < sizeof(fsync_fds)/sizeof(fsync_fds[0]); i++)
- fsync_fds[i] = -1;
-
- if (msync(index->mmap_base, index->mmap_used_length, MS_SYNC) < 0)
- return index_set_syscall_error(index, "msync()");
-
- failed = FALSE;
-
- if (index->modifylog != NULL) {
- if (!mail_modifylog_sync_file(index->modifylog, &fsync_fds[2]))
- failed = TRUE;
- }
-
- for (i = 0; i < sizeof(fsync_fds)/sizeof(fsync_fds[0]); i++) {
- if (fsync_fds[i] != -1 && fdatasync(fsync_fds[i]) < 0)
- index_set_error(index, "fdatasync(%u) failed: %m", i);
- }
-
- if (fdatasync(index->fd) < 0)
- return index_set_syscall_error(index, "fdatasync()");
-
- return !failed;
-}
-
-int mail_index_fmdatasync(struct mail_index *index, size_t size)
-{
- i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
- if (!index->anon_mmap) {
- if (msync(index->mmap_base, size, MS_SYNC) < 0)
- return index_set_syscall_error(index, "msync()");
- if (fdatasync(index->fd) < 0)
- return index_set_syscall_error(index, "fdatasync()");
- }
-
- return TRUE;
-}
-
-static void mail_index_update_header_changes(struct mail_index *index)
-{
- if (index->set_flags != 0) {
- index->header->flags |= index->set_flags;
- index->set_flags = 0;
- }
-}
-
-static int mail_index_write_header_changes(struct mail_index *index)
-{
- int failed = FALSE;
-
- /* use our own locking here so we don't mess up with any other
- index states, like inconsistency. */
- if (!mail_index_wait_lock(index, F_WRLCK))
- return FALSE;
-
-#ifdef DEBUG
- mprotect(index->mmap_base, index->mmap_used_length,
- PROT_READ|PROT_WRITE);
-#endif
-
- mail_index_update_header_changes(index);
-
- if (!index->anon_mmap) {
- if (msync(index->mmap_base, index->header_size, MS_SYNC) < 0) {
- index_set_syscall_error(index, "msync()");
- failed = TRUE;
- }
- }
-
-#ifdef DEBUG
- mprotect(index->mmap_base, index->mmap_used_length, PROT_NONE);
-#endif
-
- if (!mail_index_wait_lock(index, F_UNLCK))
- return FALSE;
-
- return !failed;
-}
-
-static int mail_index_lock_remove(struct mail_index *index)
-{
- enum mail_lock_type old_lock_type;
- int ret = TRUE;
-
- while (index->cache_later_locks > 0) {
- if (!mail_cache_unlock(index->cache))
- ret = FALSE;
- index->cache_later_locks--;
- }
-
- if (!mail_index_wait_lock(index, F_UNLCK))
- return FALSE;
-
- old_lock_type = index->lock_type;
- index->lock_type = MAIL_LOCK_UNLOCK;
-
- if (old_lock_type == MAIL_LOCK_SHARED) {
- /* releasing shared lock. we may need to update some
- flags in header. */
- unsigned int old_flags;
-
- old_flags = index->header->flags;
-
- if ((old_flags | index->set_flags) != old_flags)
- return mail_index_write_header_changes(index);
- }
-
- debug_mprotect(index->mmap_base, index->mmap_full_length, index);
- return ret;
-}
-
-static int mail_index_lock_change(struct mail_index *index,
- enum mail_lock_type lock_type, int try_lock)
-{
- int ret, fd_lock_type;
-
- /* shared -> exclusive can deadlock */
- i_assert(try_lock || lock_type != MAIL_LOCK_EXCLUSIVE ||
- index->lock_type != MAIL_LOCK_SHARED);
-
- /* locking index when cache is locked can deadlock */
- i_assert(try_lock || index->lock_type == MAIL_LOCK_EXCLUSIVE ||
- index->cache == NULL || !mail_cache_is_locked(index->cache));
-
- if (index->inconsistent) {
- /* index is in inconsistent state and nothing else than
- free() is allowed for it. */
- if (index->error == NULL) {
- index->error =
- i_strdup("Index is in inconsistent state");
- }
- return FALSE;
- }
-
- fd_lock_type = MAIL_LOCK_TO_FLOCK(lock_type);
- if (try_lock) {
- ret = file_try_lock(index->fd, fd_lock_type);
- if (ret < 0)
- index_set_syscall_error(index, "file_try_lock()");
- if (ret <= 0)
- return FALSE;
- } else {
- if (!mail_index_wait_lock(index, fd_lock_type))
- return FALSE;
- }
-
- index->lock_type = lock_type;
- debug_mprotect(index->mmap_base, index->mmap_full_length, index);
-
- if (!mail_index_mmap_update(index)) {
- (void)index->set_lock(index, MAIL_LOCK_UNLOCK);
- return FALSE;
- }
-
- if (index->indexid != index->header->indexid) {
- /* index was rebuilt, there's no way we can maintain
- consistency */
- index_set_error(index, "Warning: Inconsistency - Index "
- "%s was rebuilt while we had it open",
- index->filepath);
- index->inconsistent = TRUE;
- return FALSE;
- }
-
- if (index->header->flags & MAIL_INDEX_HDR_FLAG_FSCK) {
- /* someone just partially updated the index, need to fsck it */
- if (lock_type == MAIL_LOCK_SHARED) {
- /* we need exclusive lock so fsck()'s set_lock() won't
- get us back here */
- if (!mail_index_lock_remove(index))
- return FALSE;
-
- if (!mail_index_wait_lock(index, F_WRLCK))
- return FALSE;
- index->lock_type = MAIL_LOCK_EXCLUSIVE;
-
- debug_mprotect(index->mmap_base,
- index->mmap_full_length, index);
- }
-
- /* check again, in case it was already fscked while we had
- it unlocked for a while */
- if (index->header->flags & MAIL_INDEX_HDR_FLAG_FSCK) {
- if (!index->fsck(index))
- return FALSE;
- }
-
- if (lock_type == MAIL_LOCK_SHARED) {
- /* drop exclusive lock */
- return index->set_lock(index, lock_type);
- }
- }
-
- if (lock_type == MAIL_LOCK_EXCLUSIVE) {
- /* while holding exclusive lock, keep the FSCK flag on.
- when the lock is released, the FSCK flag will also be
- removed. */
- index->excl_lock_counter++;
- index->header->flags |= MAIL_INDEX_HDR_FLAG_FSCK;
- if (!mail_index_fmdatasync(index, index->header_size)) {
- (void)index->set_lock(index, MAIL_LOCK_UNLOCK);
- return FALSE;
- }
- }
-
- return TRUE;
-}
-
-static int mail_index_lock_full(struct mail_index *index,
- enum mail_lock_type lock_type, int try_lock)
-{
- int keep_fsck;
-
- if (index->lock_type == lock_type)
- return TRUE;
-
- if (index->lock_type == MAIL_LOCK_EXCLUSIVE) {
- index->excl_lock_counter++;
- if (index->modifylog != NULL)
- mail_modifylog_notify_lock_drop(index->modifylog);
- }
-
- if (index->anon_mmap) {
- /* anonymous mmaps are private and don't need any locking */
-#ifdef DEBUG
- mprotect(index->mmap_base, index->mmap_used_length,
- PROT_READ|PROT_WRITE);
-#endif
- mail_index_update_header_changes(index);
-
- index->lock_type = lock_type;
- debug_mprotect(index->mmap_base, index->mmap_full_length,
- index);
- return TRUE;
- }
-
- if (index->lock_type == MAIL_LOCK_EXCLUSIVE) {
- /* dropping exclusive lock (either unlock or to shared) */
- keep_fsck = (index->set_flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0;
- mail_index_update_header_changes(index);
-
- if (index->sync_dirty_stamp == 0) {
- index->header->sync_stamp = index->sync_stamp;
- index->header->sync_size = index->sync_size;
- }
-
- /* remove the FSCK flag only after successful fsync() */
- if (mail_index_sync_file(index) && !keep_fsck) {
- index->header->flags &= ~MAIL_INDEX_HDR_FLAG_FSCK;
- if (msync(index->mmap_base, index->header_size,
- MS_SYNC) < 0) {
- /* we only failed to remove the fsck flag,
- so this isn't fatal. */
- index_set_syscall_error(index, "msync()");
- }
- }
- }
-
- if (lock_type == MAIL_LOCK_UNLOCK)
- return mail_index_lock_remove(index);
- else
- return mail_index_lock_change(index, lock_type, try_lock);
-}
-
-int mail_index_set_lock(struct mail_index *index, enum mail_lock_type lock_type)
-{
- return mail_index_lock_full(index, lock_type, FALSE);
-}
-
-int mail_index_try_lock(struct mail_index *index, enum mail_lock_type lock_type)
-{
- return mail_index_lock_full(index, lock_type, TRUE);
-}
-
-void mail_index_set_lock_notify_callback(struct mail_index *index,
- mail_lock_notify_callback_t callback,
- void *context)
-{
- index->lock_notify_cb = callback;
- index->lock_notify_context = context;
-}
-
-struct mail_index_header *mail_index_get_header(struct mail_index *index)
-{
- i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
-
- return index->header;
-}
-
-void mail_index_mark_flag_changes(struct mail_index *index,
- struct mail_index_record *rec,
- enum mail_flags old_flags,
- enum mail_flags new_flags)
-{
- if ((old_flags & MAIL_SEEN) == 0 && (new_flags & MAIL_SEEN)) {
- /* unseen -> seen */
- index->header->seen_messages_count++;
- if (index->header->first_unseen_uid_lowwater == rec->uid)
- index->header->first_unseen_uid_lowwater++;
- } else if ((old_flags & MAIL_SEEN) && (new_flags & MAIL_SEEN) == 0) {
- /* seen -> unseen */
- if (index->header->seen_messages_count ==
- index->header->messages_count) {
- /* this is the first unseen message */
- index->header->first_unseen_uid_lowwater = rec->uid;
- } else if (rec->uid < index->header->first_unseen_uid_lowwater)
- index->header->first_unseen_uid_lowwater = rec->uid;
-
- if (index->header->seen_messages_count == 0) {
- index_set_corrupted(index,
- "seen_messages_count in header is invalid");
- } else {
- index->header->seen_messages_count--;
- }
- }
-
- if ((old_flags & MAIL_DELETED) == 0 && (new_flags & MAIL_DELETED)) {
- /* undeleted -> deleted */
- index->header->deleted_messages_count++;
-
- if (index->header->deleted_messages_count == 1) {
- /* this is the first deleted message */
- index->header->first_deleted_uid_lowwater = rec->uid;
- } else if (rec->uid < index->header->first_deleted_uid_lowwater)
- index->header->first_deleted_uid_lowwater = rec->uid;
- } else if ((old_flags & MAIL_DELETED) &&
- (new_flags & MAIL_DELETED) == 0) {
- /* deleted -> undeleted */
- if (index->header->first_deleted_uid_lowwater == rec->uid)
- index->header->first_deleted_uid_lowwater++;
- if (index->header->deleted_messages_count == 0) {
- index_set_corrupted(index,
- "deleted_messages_count in header is invalid");
- } else {
- index->header->deleted_messages_count--;
- }
- }
-}
-
-#define INDEX_NEED_COMPRESS(records, hdr) \
- ((records) > INDEX_MIN_RECORDS_COUNT && \
- (records) * (100-INDEX_COMPRESS_PERCENTAGE) / 100 > \
- (hdr)->messages_count)
-
-int mail_index_expunge(struct mail_index *index,
- struct mail_index_record *first_rec,
- struct mail_index_record *last_rec,
- unsigned int first_seq, unsigned int last_seq,
- int external_change)
-{
- unsigned int first_uid, last_uid;
-
- i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
- i_assert(first_seq != 0);
- i_assert(first_seq <= last_seq);
-
- index->expunge_counter++;
-
- first_uid = first_rec->uid;
- last_uid = last_rec->uid;
-
- if (!mail_index_expunge_record_range(index, first_rec, last_rec))
- return FALSE;
-
- if (index->modifylog != NULL) {
- if (!mail_modifylog_add_expunges(index->modifylog,
- first_seq, last_seq,
- first_uid, last_uid,
- external_change))
- return FALSE;
- }
-
- if (index->header->messages_count == 0) {
- /* all mail was deleted, truncate cache file */
- if (!mail_cache_truncate(index->cache))
- return FALSE;
- }
-
- return TRUE;
-}
-
-int mail_index_update_flags(struct mail_index *index,
- struct mail_index_record *rec, unsigned int seq,
- enum modify_type modify_type, enum mail_flags flags,
- int external_change)
-{
- enum mail_flags new_flags;
-
- i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
- i_assert(seq != 0);
-
- switch (modify_type) {
- case MODIFY_ADD:
- new_flags = rec->msg_flags | flags;
- break;
- case MODIFY_REMOVE:
- new_flags = rec->msg_flags & ~flags;
- break;
- case MODIFY_REPLACE:
- new_flags = flags;
- break;
- default:
- new_flags = 0;
- i_unreached();
- }
-
- if (new_flags == rec->msg_flags)
- return TRUE; /* no changes */
-
- mail_index_mark_flag_changes(index, rec, rec->msg_flags, new_flags);
-
- rec->msg_flags = new_flags;
- return index->modifylog == NULL ? TRUE :
- mail_modifylog_add_flags(index->modifylog, seq,
- rec->uid, external_change);
-}
-
-static int mail_index_grow(struct mail_index *index)
-{
- uoff_t pos;
- unsigned int grow_count;
- void *base;
-
- grow_count = index->header->messages_count *
- INDEX_GROW_PERCENTAGE / 100;
- if (grow_count < 16)
- grow_count = 16;
-
- pos = index->mmap_full_length +
- (grow_count * sizeof(struct mail_index_record));
- i_assert(pos < OFF_T_MAX);
-
- if (index->anon_mmap) {
- i_assert(pos < SSIZE_T_MAX);
-
- base = mremap_anon(index->mmap_base, index->mmap_full_length,
- (size_t)pos, MREMAP_MAYMOVE);
- if (base == MAP_FAILED)
- return index_set_syscall_error(index, "mremap_anon()");
-
- index->mmap_base = base;
- index->mmap_full_length = (size_t)pos;
- return mmap_verify(index);
- }
-
- if (file_set_size(index->fd, (off_t)pos) < 0)
- return index_set_syscall_error(index, "file_set_size()");
-
- /* file size changed, let others know about it too by changing
- sync_id in header. */
- index->header->master_sync_id++;
-
- if (!mail_index_mmap_update(index))
- return FALSE;
-
- return TRUE;
-}
-
-struct mail_index_record *mail_index_append(struct mail_index *index)
-{
- struct mail_index_record *rec;
-
- i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
- if (index->header->next_uid == MAX_ALLOWED_UID) {
- index->set_flags |= MAIL_INDEX_HDR_FLAG_REBUILD;
- index_set_error(index, "Reached maximum UID in mailbox %s, "
- "rebuilding index", index->filepath);
- return NULL;
- }
-
- if (index->mmap_used_length == index->mmap_full_length) {
- if (!mail_index_grow(index))
- return NULL;
- }
-
- i_assert(index->header->used_file_size == index->mmap_used_length);
- i_assert(index->mmap_used_length + sizeof(struct mail_index_record) <=
- index->mmap_full_length);
-
- index->header->messages_count++;
-
- rec = (struct mail_index_record *) ((char *) index->mmap_base +
- index->mmap_used_length);
- rec->uid = index->header->next_uid++;
- rec->msg_flags = 0;
- rec->cache_offset = 0;
-
- index->header->used_file_size += sizeof(*rec);
- index->mmap_used_length += sizeof(*rec);
-
- return rec;
-}
-
-enum mail_index_error mail_index_get_last_error(struct mail_index *index)
-{
- if (index->inconsistent)
- return MAIL_INDEX_ERROR_INCONSISTENT;
- if (index->nodiskspace)
- return MAIL_INDEX_ERROR_DISKSPACE;
- if (index->index_lock_timeout)
- return MAIL_INDEX_ERROR_INDEX_LOCK_TIMEOUT;
- if (index->mailbox_lock_timeout)
- return MAIL_INDEX_ERROR_MAILBOX_LOCK_TIMEOUT;
-
- if (index->error != NULL)
- return MAIL_INDEX_ERROR_INTERNAL;
-
- return MAIL_INDEX_ERROR_NONE;
-}
-
-const char *mail_index_get_last_error_text(struct mail_index *index)
-{
- return index->error;
-}
+++ /dev/null
-#ifndef __MAIL_INDEX_H
-#define __MAIL_INDEX_H
-
-#include "file-dotlock.h"
-#include "message-parser.h"
-#include "imap-util.h"
-
-#define MAIL_INDEX_MAJOR_VERSION 3
-#define MAIL_INDEX_MINOR_VERSION 0
-
-#define INDEX_FILE_PREFIX ".imap.index"
-
-enum mail_index_open_flags {
- /* Create index if it doesn't exist */
- MAIL_INDEX_OPEN_FLAG_CREATE = 0x01,
- /* Update \Recent flag counters */
- MAIL_INDEX_OPEN_FLAG_UPDATE_RECENT = 0x02,
- /* Compressing and cache updates are not performed */
- MAIL_INDEX_OPEN_FLAG_FAST = 0x04,
- /* Invalidate memory maps before accessing them */
- MAIL_INDEX_OPEN_FLAG_MMAP_INVALIDATE = 0x08,
-
- /* internal: we're creating the index */
- _MAIL_INDEX_OPEN_FLAG_CREATING = 0x100
-};
-
-enum mail_index_header_compat_flags {
- MAIL_INDEX_COMPAT_LITTLE_ENDIAN = 0x01
-};
-
-enum mail_index_header_flag {
- /* Rebuild flag is set while index is being rebuilt or when
- some error is noticed in the index file. If this flag is set,
- the index shouldn't be used before rebuilding it. */
- MAIL_INDEX_HDR_FLAG_FSCK = 0x0001,
- MAIL_INDEX_HDR_FLAG_REBUILD = 0x0002,
- MAIL_INDEX_HDR_FLAG_COMPRESS = 0x0004,
- MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE = 0x0008,
- MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES = 0x0010,
- MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS = 0x0020,
- MAIL_INDEX_HDR_FLAG_MAILDIR_NEW = 0x0040
-};
-
-enum mail_index_record_flag {
- /* If binary flags are set, it's not checked whether mail is
- missing CRs. So this flag may be set as an optimization for
- regular non-binary mails as well if it's known that it contains
- valid CR+LF line breaks. */
- MAIL_INDEX_FLAG_BINARY_HEADER = 0x0001,
- MAIL_INDEX_FLAG_BINARY_BODY = 0x0002,
-
- /* Mail flags have been changed in index, but not written into
- actual mailbox yet. */
- MAIL_INDEX_FLAG_DIRTY = 0x0004,
-
- /* Maildir: Mail file is in new/ dir instead of cur/ */
- MAIL_INDEX_FLAG_MAILDIR_NEW = 0x0008,
-
- /* Mail header or body is known to contain NUL characters. */
- MAIL_INDEX_FLAG_HAS_NULS = 0x0010,
- /* Mail header or body is known to not contain NUL characters. */
- MAIL_INDEX_FLAG_HAS_NO_NULS = 0x0020
-};
-
-enum mail_lock_type {
- MAIL_LOCK_UNLOCK = 0,
- MAIL_LOCK_SHARED,
- MAIL_LOCK_EXCLUSIVE
-};
-
-enum mail_lock_notify_type {
- /* Mailbox is locked, will abort in secs_left */
- MAIL_LOCK_NOTIFY_MAILBOX_ABORT,
- /* Mailbox lock looks stale, will override in secs_left */
- MAIL_LOCK_NOTIFY_MAILBOX_OVERRIDE,
- /* Index is locked, will abort in secs_left */
- MAIL_LOCK_NOTIFY_INDEX_ABORT
-};
-
-enum mail_index_error {
- /* No errors */
- MAIL_INDEX_ERROR_NONE,
- /* Internal error, see get_error_text() for more information. */
- MAIL_INDEX_ERROR_INTERNAL,
- /* Index is now in inconsistent state with the previous known state,
- meaning that the message IDs etc. may have changed - only way to
- recover this would be to fully close the mailbox and reopen it.
- With IMAP this would mean a forced disconnection since we can't do
- forced CLOSE. */
- MAIL_INDEX_ERROR_INCONSISTENT,
- /* We ran out of available disk space. */
- MAIL_INDEX_ERROR_DISKSPACE,
- /* Mail index locking timeouted */
- MAIL_INDEX_ERROR_INDEX_LOCK_TIMEOUT,
- /* Mailbox locking timeouted */
- MAIL_INDEX_ERROR_MAILBOX_LOCK_TIMEOUT
-};
-
-typedef void mail_lock_notify_callback_t(enum mail_lock_notify_type notify_type,
- unsigned int secs_left, void *context);
-
-struct mail_index_header {
- /* major version is increased only when you can't have backwards
- compatibility. minor version is increased when header size is
- increased to contain new non-critical fields. */
- uint8_t major_version;
- uint8_t minor_version;
- uint8_t header_size;
- uint8_t reserved;
-
- /* 0 = flags
- 1 = sizeof(uoff_t)
- 2 = sizeof(time_t)
- 3 = reserved, 0 for now */
- uint8_t compat_data[4];
-
- uint32_t indexid;
- uint32_t used_file_size;
-
- /* file needs to be reopened if sync_ids change. */
- uint32_t master_sync_id;
- uint32_t cache_sync_id;
- uint32_t log_sync_id;
-
- uint32_t flags;
-
- uint32_t uid_validity;
- uint32_t next_uid;
-
- uint32_t messages_count;
- uint32_t seen_messages_count;
- uint32_t deleted_messages_count;
- uint32_t last_nonrecent_uid;
-
- /* these UIDs may not exist and may not even be unseen */
- uint32_t first_unseen_uid_lowwater;
- uint32_t first_deleted_uid_lowwater;
-
- uint64_t sync_size;
- uint32_t sync_stamp;
-};
-
-struct mail_index_record {
- uint32_t uid;
- uint32_t msg_flags;
- uint32_t cache_offset;
-};
-
-struct mail_index {
- /* Note that opening same index twice in the same process is a bad
- idea since they share the same file locks. As soon one of the
- indexes is closed, the locks in second index are dropped which
- especially hurts modify log since it keeps locks all the time. */
- int (*open)(struct mail_index *index, enum mail_index_open_flags flags);
-
- /* Free index from memory. */
- void (*free)(struct mail_index *index);
-
- /* Lock/unlock index. May block. Note that unlocking must not
- reset error from get_last_error() as unlocking can be done as
- a cleanup after some other function failed. Index is always
- mmap()ed after set_lock() succeeds.
-
- Trying to change a shared lock into exclusive lock is a fatal
- error, since it may create a deadlock. Even though operating
- system should detect it and fail, it's not a good idea to even
- let it happen. Better ways to do this would be to a) mark the
- data to be updated later, b) use try_lock() if the update is
- preferred but not required, c) unlock + lock again, but make
- sure that won't create race conditions. */
- int (*set_lock)(struct mail_index *index,
- enum mail_lock_type lock_type);
-
- /* Try locking the index. Returns TRUE if the lock was got and
- FALSE if lock isn't possible to get currently or some other error
- occured. Never blocks. */
- int (*try_lock)(struct mail_index *index,
- enum mail_lock_type lock_type);
-
- /* If we have to wait for the lock, the given lock notify function
- is called once in a while. */
- void (*set_lock_notify_callback)(struct mail_index *index,
- mail_lock_notify_callback_t *callback,
- void *context);
-
- /* Rebuild the whole index. Note that this changes the indexid
- so all the other files must also be rebuilt after this call.
- Index MUST NOT have shared lock, but exclusive lock or no lock at
- all is fine. Note that this function may leave the index
- exclusively locked, and always sets index->inconsistent = TRUE. */
- int (*rebuild)(struct mail_index *index);
-
- /* Verify that the index is valid. If anything invalid is found,
- index is set inconsistent and to be rebuilt at next open.
- Same locking issues as with rebuild(). */
- int (*fsck)(struct mail_index *index);
-
- /* Synchronize the index with the mailbox. Index must not have shared
- lock when calling this function. The data_lock_type specifies what
- lock should be set to data file (mbox file). This function may
- leave the index in ANY locking state. If changes is non-NULL, it's
- set to TRUE if any changes were noticed. If minimal_sync is TRUE,
- we do as little as possible to get data file locked (ie. noop with
- maildir). */
- int (*sync_and_lock)(struct mail_index *index, int minimal_sync,
- enum mail_lock_type data_lock_type, int *changes);
-
- /* Returns the index header (never fails). The index needs to be
- locked before calling this function, and must be kept locked as
- long as you keep using the returned structure. */
- struct mail_index_header *(*get_header)(struct mail_index *index);
-
- /* sequence -> data lookup. The index needs to be locked before calling
- this function, and must be kept locked as long as you keep using
- the returned structure. */
- struct mail_index_record *(*lookup)(struct mail_index *index,
- unsigned int seq);
-
- /* Return the next record after specified record, or NULL if it was
- last record. The index must be locked all the time between
- lookup() and last next() call. rec must not have been expunged. */
- struct mail_index_record *(*next)(struct mail_index *index,
- struct mail_index_record *rec);
-
- /* Find first existing UID in range. Sequence number is also retrieved
- if seq_r is non-NULL. */
- struct mail_index_record *(*lookup_uid_range)(struct mail_index *index,
- unsigned int first_uid,
- unsigned int last_uid,
- unsigned int *seq_r);
-
- /* Open mail file and return it as mmap()ed IStream. If we fail,
- we return NULL and set deleted = TRUE if failure was because the
- mail was just deleted (ie. not an error). received_date is set
- if it's non-NULL. */
- struct istream *(*open_mail)(struct mail_index *index,
- struct mail_index_record *rec,
- time_t *received_date, int *deleted);
-
- /* Returns received date of message, or (time_t)-1 if error occured. */
- time_t (*get_received_date)(struct mail_index *index,
- struct mail_index_record *rec);
-
- /* Expunge mails from index. Modifylog is also updated. The
- index must be exclusively locked before calling this function.
-
- first_rec+1 .. last_rec-1 range may contain already expunged
- records.
-
- Note that all record pointers are invalidated after this call as
- expunging may radically modify the file. */
- int (*expunge)(struct mail_index *index,
- struct mail_index_record *first_rec,
- struct mail_index_record *last_rec,
- unsigned int first_seq, unsigned int last_seq,
- int external_change);
-
- /* Update mail flags. The index must be exclusively locked before
- calling this function. */
- int (*update_flags)(struct mail_index *index,
- struct mail_index_record *rec, unsigned int seq,
- enum modify_type modify_type, enum mail_flags flags,
- int external_change);
-
- /* Append a new record to index. The index must be exclusively
- locked before calling this function. */
- struct mail_index_record *(*append)(struct mail_index *index);
-
- /* Returns the last error code. */
- enum mail_index_error (*get_last_error)(struct mail_index *index);
-
- /* Returns the full error message for last error. This message may
- contain paths etc. so it shouldn't be shown to users. */
- const char *(*get_last_error_text)(struct mail_index *index);
-
-/* private: */
- struct mail_cache *cache;
- struct mail_modify_log *modifylog;
- struct mail_custom_flags *custom_flags;
-
- char *dir; /* directory where to place the index files */
- char *filepath; /* index file path */
- char *mailbox_path; /* file/directory for mailbox location */
- char *control_dir; /* destination for control files */
- unsigned int indexid;
- unsigned int master_sync_id, cache_sync_id, log_sync_id;
-
- /* updated whenever exclusive lock is set/unset */
- unsigned int excl_lock_counter;
- /* updated whenever expunge() is called */
- unsigned int expunge_counter;
-
- int mbox_fd;
- struct istream *mbox_stream;
- enum mail_lock_type mbox_lock_type;
- struct dotlock mbox_dotlock;
-
- /* these counters can be used to check that we've synced the mailbox
- after locking it */
- unsigned int mbox_lock_counter;
- unsigned int mbox_sync_counter;
-
- /* last mbox sync: */
- dev_t mbox_dev;
- ino_t mbox_ino;
-
- /* last maildir sync: */
- time_t last_new_mtime, last_uidlist_mtime;
- int maildir_lock_fd;
- pool_t new_filename_pool;
- struct hash_table *new_filenames;
-
- int fd; /* opened index file */
- char *error; /* last error message */
-
- void *mmap_base;
- size_t mmap_used_length;
- size_t mmap_full_length;
-
- struct mail_index_header *header;
- size_t header_size;
-
- enum mail_lock_type lock_type;
- time_t sync_stamp, sync_dirty_stamp;
- uoff_t sync_size;
- time_t next_dirty_flags_flush;
- unsigned int first_recent_uid;
-
- mail_lock_notify_callback_t *lock_notify_cb;
- void *lock_notify_context;
-
- mode_t mail_create_mode;
- unsigned int private_flags_mask;
-
- /* these fields are OR'ed to the fields in index header once we
- get around grabbing exclusive lock */
- unsigned int set_flags;
- unsigned int cache_later_locks;
-
- unsigned int anon_mmap:1;
- unsigned int mmap_invalidate:1;
- unsigned int mbox_rewritten:1;
- unsigned int opened:1;
- unsigned int rebuilding:1;
- unsigned int mail_read_mmaped:1;
- unsigned int inconsistent:1;
- unsigned int nodiskspace:1;
- unsigned int index_lock_timeout:1;
- unsigned int allow_new_custom_flags:1;
- unsigned int mailbox_readonly:1;
- unsigned int mailbox_lock_timeout:1;
- unsigned int maildir_keep_new:1;
- unsigned int maildir_have_new:1;
- unsigned int maildir_synced_once:1;
-};
-
-#ifdef DEV_T_STRUCT
-/* we can't initialize dev_t as 0, and we don't know what it actually
- contains, so don't initialize them. gcc's -W option should be disabled
- with this or we get warnings.. */
-# define MAIL_INDEX_PRIVATE_FILL 0
-#else
-/* needed to remove annoying warnings about not initializing all struct
- members.. */
-#define MAIL_INDEX_PRIVATE_FILL \
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
- 0, 0, 0, 0, 0, 0, { 0, 0, 0 }, 0, \
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
- 0, 0
-#endif
-
-/* defaults - same as above but prefixed with mail_index_. */
-int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags);
-int mail_index_set_lock(struct mail_index *index,
- enum mail_lock_type lock_type);
-int mail_index_try_lock(struct mail_index *index,
- enum mail_lock_type lock_type);
-void mail_index_set_lock_notify_callback(struct mail_index *index,
- mail_lock_notify_callback_t *callback,
- void *context);
-int mail_index_fsck(struct mail_index *index);
-struct mail_index_header *mail_index_get_header(struct mail_index *index);
-struct mail_index_record *mail_index_lookup(struct mail_index *index,
- unsigned int seq);
-struct mail_index_record *mail_index_next(struct mail_index *index,
- struct mail_index_record *rec);
-struct mail_index_record *
-mail_index_lookup_uid_range(struct mail_index *index, unsigned int first_uid,
- unsigned int last_uid, unsigned int *seq_r);
-int mail_index_expunge(struct mail_index *index,
- struct mail_index_record *first_rec,
- struct mail_index_record *last_rec,
- unsigned int first_seq, unsigned int last_seq,
- int external_change);
-int mail_index_update_flags(struct mail_index *index,
- struct mail_index_record *rec, unsigned int seq,
- enum modify_type modify_type, enum mail_flags flags,
- int external_change);
-struct mail_index_record *mail_index_append(struct mail_index *index);
-enum mail_index_error mail_index_get_last_error(struct mail_index *index);
-const char *mail_index_get_last_error_text(struct mail_index *index);
-
-/* INTERNAL: */
-void mail_index_init(struct mail_index *index, const char *dir);
-int mail_index_mmap_update(struct mail_index *index);
-void mail_index_init_header(struct mail_index_header *hdr);
-void mail_index_close(struct mail_index *index);
-int mail_index_fmdatasync(struct mail_index *index, size_t size);
-void mail_index_mark_flag_changes(struct mail_index *index,
- struct mail_index_record *rec,
- enum mail_flags old_flags,
- enum mail_flags new_flags);
-int mail_index_rebuild(struct mail_index *index);
-int mail_index_compress(struct mail_index *index);
-int mail_index_truncate(struct mail_index *index);
-int mail_index_expunge_record_range(struct mail_index *index,
- struct mail_index_record *first_rec,
- struct mail_index_record *last_rec);
-
-/* Maximum allowed UID number. */
-#define MAX_ALLOWED_UID 4294967295U /* 2^32 - 1 */
-
-/* Max. mmap()ed size for a message */
-#define MAIL_MMAP_BLOCK_SIZE (1024*256)
-/* Block size when read()ing message. */
-#define MAIL_READ_BLOCK_SIZE (1024*8)
-
-/* Delete unused non-local temp files after 24h. Just to be sure we don't
- delete it too early. The temp files don't harm much anyway. */
-#define TEMP_FILE_TIMEOUT (60*24)
-
-/* number of records to always keep allocated in index file,
- either used or unused */
-#define INDEX_MIN_RECORDS_COUNT 64
-/* when empty space in index file gets full, grow the file n% larger */
-#define INDEX_GROW_PERCENTAGE 10
-/* ftruncate() the index file when only n% of it is in use */
-#define INDEX_TRUNCATE_PERCENTAGE 30
-/* don't truncate whole file anyway, keep n% of the empty space */
-#define INDEX_TRUNCATE_KEEP_PERCENTAGE 10
-/* Compress the file when deleted space reaches n% of total size */
-#define INDEX_COMPRESS_PERCENTAGE 50
-/* Compress the file when searching deleted records tree has to go this deep */
-#define INDEX_COMPRESS_DEPTH 10
-
-/* uoff_t to index file for given record */
-#define INDEX_FILE_POSITION(index, ptr) \
- ((uoff_t) ((char *) (ptr) - (char *) ((index)->mmap_base)))
-
-/* record for given index */
-#define INDEX_RECORD_AT(index, idx) \
- ((struct mail_index_record *) \
- ((char *) index->mmap_base + (index)->header_size) + (idx))
-
-/* returns the next record after last one */
-#define INDEX_END_RECORD(index) \
- ((struct mail_index_record *) \
- ((char *) (index)->mmap_base + (index)->mmap_used_length))
-
-/* index number for uoff_t position */
-#define INDEX_POSITION_INDEX(index, pos) \
- (((pos) - (index)->header_size) / \
- sizeof(struct mail_index_record))
-
-/* index number for given record */
-#define INDEX_RECORD_INDEX(index, ptr) \
- INDEX_POSITION_INDEX(index, INDEX_FILE_POSITION(index, ptr))
-
-/* mark the index corrupted */
-#define INDEX_MARK_CORRUPTED(index) \
- STMT_START { \
- (index)->set_flags |= MAIL_INDEX_HDR_FLAG_REBUILD; \
- } STMT_END
-
-/* get number of records in mmaped index */
-#define MAIL_INDEX_RECORD_COUNT(index) \
- ((index->mmap_used_length - (index)->header_size) / \
- sizeof(struct mail_index_record))
-
-/* minimum size for index file */
-#define INDEX_FILE_MIN_SIZE(index) \
- ((index)->header_size + \
- INDEX_MIN_RECORDS_COUNT * sizeof(struct mail_index_record))
-
-/* enum mail_lock_type to fcntl() lock type */
-#define MAIL_LOCK_TO_FLOCK(lock_type) \
- ((lock_type) == MAIL_LOCK_EXCLUSIVE ? F_WRLCK : \
- (lock_type) == MAIL_LOCK_SHARED ? F_RDLCK : F_UNLCK)
-
-#define INDEX_IS_IN_MEMORY(index) \
- ((index)->anon_mmap)
-
-#endif
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "buffer.h"
-#include "file-lock.h"
-#include "file-set-size.h"
-#include "mmap-util.h"
-#include "write-full.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-#include "mail-modifylog.h"
-
-#include <stddef.h>
-#include <stdlib.h>
-#include <fcntl.h>
-
-/* Maximum size for modify log (isn't exact) */
-#define MAX_MODIFYLOG_SIZE (4096*8)
-
-/* How large chunks to use to grow log file */
-#define MODIFYLOG_GROW_SIZE (sizeof(struct modify_log_record) * 128)
-
-#define MODIFY_LOG_INITIAL_SIZE \
- (sizeof(struct modify_log_header) + MODIFYLOG_GROW_SIZE)
-
-#define MODIFYLOG_FILE_POSITION(log, ptr) \
- ((size_t) ((char *) (ptr) - (char *) (log)->mmap_base))
-
-/* FIXME: not ANSI-C */
-#define IS_PTR_IN_RANGE(ptr, area_ptr, area_size) \
- ((char *) (ptr) >= (char *) (area_ptr) && \
- (char *) (ptr) < (char *) (area_ptr) + (area_size))
-
-struct modify_log_file {
- struct mail_modify_log *log;
-
- int fd;
- char *filepath;
-
- void *mmap_base;
- size_t mmap_used_length;
- size_t mmap_full_length;
-
- struct modify_log_record *last_expunge, *last_flags;
- int last_expunge_external, last_flags_external;
-
- struct modify_log_header *header;
- uoff_t synced_position;
- unsigned int synced_id;
-
- unsigned int anon_mmap:1;
- unsigned int modified:1;
- unsigned int second_log:1;
-};
-
-struct mail_modify_log {
- struct mail_index *index;
-
- struct modify_log_record *iterator_end;
-
- struct modify_log_file file1, file2;
- struct modify_log_file *head, *tail;
-
- int cache_have_others;
- unsigned int cache_lock_counter;
-};
-
-static const struct modify_log_expunge no_expunges = { 0, 0, 0 };
-
-static int modifylog_set_syscall_error(struct modify_log_file *file,
- const char *function)
-{
- i_assert(function != NULL);
-
- if (ENOSPACE(errno)) {
- file->log->index->nodiskspace = TRUE;
- return FALSE;
- }
-
- index_set_error(file->log->index,
- "%s failed with modify log file %s: %m",
- function, file->filepath);
- return FALSE;
-}
-
-static int modifylog_set_corrupted(struct modify_log_file *file,
- const char *fmt, ...)
-{
- va_list va;
-
- va_start(va, fmt);
- t_push();
-
- index_set_error(file->log->index, "Corrupted modify log file %s: %s",
- file->filepath, t_strdup_vprintf(fmt, va));
-
- t_pop();
- va_end(va);
-
- /* make sure we don't get back here */
- file->log->index->inconsistent = TRUE;
- (void)unlink(file->filepath);
-
- return FALSE;
-}
-
-static int modifylog_drop_lock(struct modify_log_file *file)
-{
- int ret;
-
- /* revert back to shared lock */
- ret = file_try_lock(file->fd, F_RDLCK);
- if (ret < 0) {
- modifylog_set_syscall_error(file, "file_try_lock()");
- return -1;
- }
-
- if (ret == 0) {
- /* shouldn't happen */
- index_set_error(file->log->index,
- "file_try_lock(F_WRLCK -> F_RDLCK) "
- "failed with file %s", file->filepath);
- return -1;
- }
-
- return 1;
-}
-
-static int modifylog_file_have_other_users(struct modify_log_file *file,
- int keep_lock)
-{
- int ret;
-
- if (file->anon_mmap)
- return 0;
-
- /* try grabbing exclusive lock */
- ret = file_try_lock(file->fd, F_WRLCK);
- if (ret <= 0) {
- if (ret < 0)
- modifylog_set_syscall_error(file, "file_try_lock()");
- return ret < 0 ? -1 : 1;
- }
-
- if (keep_lock)
- return 0;
- else
- return modifylog_drop_lock(file) < 0 ? -1 : 0;
-}
-
-/* returns 1 = yes, 0 = no, -1 = error */
-static int modifylog_have_other_users(struct mail_modify_log *log,
- int keep_lock)
-{
- struct modify_log_file *file;
- int ret;
-
- ret = modifylog_file_have_other_users(log->head, keep_lock);
- if (ret == 0) {
- /* check the other file too */
- file = log->head == &log->file1 ? &log->file2 : &log->file1;
-
- ret = file->fd == -1 ? 0 :
- modifylog_file_have_other_users(file, FALSE);
- if (keep_lock && ret != 0) {
- if (modifylog_drop_lock(log->head) < 0)
- return -1;
- }
- }
-
- return ret;
-}
-
-static int mmap_update(struct modify_log_file *file, int forced)
-{
- struct modify_log_header *hdr;
- unsigned int extra;
-
- if (file->log->index->mmap_invalidate && file->mmap_base != NULL) {
- if (msync(file->mmap_base, file->mmap_used_length,
- MS_SYNC | MS_INVALIDATE) < 0)
- return modifylog_set_syscall_error(file, "msync()");
- }
-
- if (!forced && file->header != NULL &&
- file->mmap_full_length >= file->header->used_file_size) {
- file->mmap_used_length = file->header->used_file_size;
- debug_mprotect(file->mmap_base, file->mmap_full_length,
- file->log->index);
- return TRUE;
- }
-
- i_assert(!file->anon_mmap);
-
- if (file->mmap_base != NULL) {
- /* make sure we're synced before munmap() */
- if (file->modified &&
- msync(file->mmap_base, file->mmap_used_length, MS_SYNC) < 0)
- return modifylog_set_syscall_error(file, "msync()");
- file->modified = FALSE;
-
- if (munmap(file->mmap_base, file->mmap_full_length) < 0)
- modifylog_set_syscall_error(file, "munmap()");
- }
-
- file->log->iterator_end = NULL;
-
- file->mmap_used_length = 0;
- file->header = NULL;
-
- file->last_expunge = NULL;
- file->last_flags = NULL;
-
- file->mmap_base = mmap_rw_file(file->fd, &file->mmap_full_length);
- if (file->mmap_base == MAP_FAILED) {
- file->mmap_base = NULL;
- return modifylog_set_syscall_error(file, "mmap()");
- }
-
- if (file->mmap_full_length < sizeof(struct modify_log_header)) {
- index_set_error(file->log->index, "Too small modify log %s",
- file->filepath);
- (void)unlink(file->filepath);
- return FALSE;
- }
-
- extra = (file->mmap_full_length - sizeof(struct modify_log_header)) %
- sizeof(struct modify_log_record);
-
- if (extra != 0) {
- /* partial write or corrupted -
- truncate the file to valid length */
- file->mmap_full_length -= extra;
- if (ftruncate(file->fd, (off_t)file->mmap_full_length) < 0)
- modifylog_set_syscall_error(file, "ftruncate()");
- }
-
- hdr = file->mmap_base;
- if (hdr->used_file_size > file->mmap_full_length) {
- modifylog_set_corrupted(file,
- "used_file_size larger than real file size "
- "(%"PRIuUOFF_T" vs %"PRIuSIZE_T")",
- hdr->used_file_size, file->mmap_full_length);
- return FALSE;
- }
-
- if (hdr->used_file_size < sizeof(struct modify_log_header) ||
- (hdr->used_file_size - sizeof(struct modify_log_header)) %
- sizeof(struct modify_log_record) != 0) {
- modifylog_set_corrupted(file,
- "Invalid used_file_size in header (%"PRIuUOFF_T")",
- hdr->used_file_size);
- return FALSE;
- }
-
- file->header = file->mmap_base;
- file->mmap_used_length = hdr->used_file_size;
- debug_mprotect(file->mmap_base, file->mmap_full_length,
- file->log->index);
- return TRUE;
-}
-
-static int mmap_init_update(struct modify_log_file *file)
-{
- if (!mmap_update(file, TRUE))
- return FALSE;
-
- file->synced_id = file->header->sync_id;
- file->synced_position = file->mmap_used_length;
- return TRUE;
-}
-
-static struct mail_modify_log *mail_modifylog_new(struct mail_index *index)
-{
- struct mail_modify_log *log;
-
- log = i_new(struct mail_modify_log, 1);
- log->index = index;
-
- log->file1.fd = -1;
- log->file2.fd = -1;
-
- log->file1.log = log;
- log->file2.log = log;
-
- log->file1.filepath = i_strconcat(index->filepath, ".log", NULL);
- log->file2.filepath = i_strconcat(index->filepath, ".log.2", NULL);
-
- index->modifylog = log;
- return log;
-}
-
-static void modifylog_munmap(struct modify_log_file *file)
-{
- if (file->anon_mmap) {
- if (munmap_anon(file->mmap_base, file->mmap_full_length) < 0)
- modifylog_set_syscall_error(file, "munmap_anon()");
- } else if (file->mmap_base != NULL) {
- if (munmap(file->mmap_base, file->mmap_full_length) < 0)
- modifylog_set_syscall_error(file, "munmap()");
- }
- file->mmap_base = NULL;
- file->mmap_full_length = 0;
- file->mmap_used_length = 0;
- file->header = NULL;
-
- file->last_expunge = NULL;
- file->last_flags = NULL;
-}
-
-static void modifylog_close_file(struct modify_log_file *file)
-{
- modifylog_munmap(file);
-
- if (file->fd != -1) {
- if (close(file->fd) < 0)
- modifylog_set_syscall_error(file, "close()");
- file->fd = -1;
- }
-}
-
-static void mail_modifylog_init_header(struct mail_modify_log *log,
- struct modify_log_header *hdr)
-{
- memset(hdr, 0, sizeof(struct modify_log_header));
- hdr->indexid = log->index->indexid;
- hdr->used_file_size = sizeof(struct modify_log_header);
-}
-
-static int mail_modifylog_init_fd(struct modify_log_file *file, int fd)
-{
- struct modify_log_header hdr;
-
- mail_modifylog_init_header(file->log, &hdr);
-
- if (write_full(fd, &hdr, sizeof(hdr)) < 0)
- return modifylog_set_syscall_error(file, "write_full()");
-
- if (file_set_size(fd, MODIFY_LOG_INITIAL_SIZE) < 0)
- return modifylog_set_syscall_error(file, "file_set_size()");
-
- return TRUE;
-}
-
-static int modifylog_mark_full(struct modify_log_file *file)
-{
- unsigned int sync_id = SYNC_ID_FULL;
-
- if (file->mmap_base != NULL) {
- file->header->sync_id = SYNC_ID_FULL;
-
- if (msync(file->mmap_base, sizeof(struct modify_log_header),
- MS_SYNC) < 0)
- return modifylog_set_syscall_error(file, "msync()");
- } else {
- off_t offset = offsetof(struct modify_log_header, sync_id);
-
- if (lseek(file->fd, offset, SEEK_SET) < 0)
- return modifylog_set_syscall_error(file, "lseek()");
-
- if (write_full(file->fd, &sync_id, sizeof(sync_id)) < 0) {
- modifylog_set_syscall_error(file, "write_full()");
- return FALSE;
- }
- }
-
- return TRUE;
-}
-
-/* Returns 1 = ok, 0 = can't lock file, -1 = error */
-static int modifylog_reuse_or_create_file(struct modify_log_file *file)
-{
- struct mail_index *index = file->log->index;
- int fd, ret;
-
- if (INDEX_IS_IN_MEMORY(index))
- return -1;
-
- fd = open(file->filepath, O_RDWR | O_CREAT, 0660);
- if (fd == -1) {
- modifylog_set_syscall_error(file, "open()");
- return -1;
- }
-
- /* 1) there's race condition between open() and file_try_lock(), so
- if we can't get a lock it means the other process did
- 2) this function is also called by try_switch_log() which uses
- this check to make sure it's not locked by others. */
- ret = file_try_lock(fd, F_WRLCK);
- if (ret < 0)
- modifylog_set_syscall_error(file, "file_try_lock()");
-
- if (ret > 0 && mail_modifylog_init_fd(file, fd)) {
- /* drop back to read lock */
- if (file_try_lock(fd, F_RDLCK) <= 0) {
- modifylog_set_syscall_error(file, "file_try_lock()");
- ret = -1;
- }
-
- if (ret > 0) {
- file->fd = fd;
- return 1;
- }
- }
-
- if (close(fd) < 0)
- modifylog_set_syscall_error(file, "close()");
- return ret;
-}
-
-/* Returns 1 = ok, 0 = full, -1 = error */
-static int mail_modifylog_open_and_verify(struct modify_log_file *file)
-{
- struct mail_index *index = file->log->index;
- struct modify_log_header hdr;
- ssize_t ret;
- int fd;
-
- fd = open(file->filepath, O_RDWR);
- if (fd == -1) {
- if (errno != ENOENT)
- modifylog_set_syscall_error(file, "open()");
- return -1;
- }
-
- if (file_wait_lock(fd, F_RDLCK) <= 0) {
- modifylog_set_syscall_error(file, "file_wait_lock()");
- (void)close(fd);
- return -1;
- }
-
- ret = read(fd, &hdr, sizeof(hdr));
- if (ret < 0)
- modifylog_set_syscall_error(file, "read()");
- else if (ret != sizeof(hdr)) {
- index_set_error(index, "Corrupted modify log %s: "
- "File too small", file->filepath);
- ret = -1;
-
- (void)unlink(file->filepath);
- } else {
- ret = 1;
- }
-
- if (ret > 0 && hdr.indexid != index->indexid) {
- index_set_error(index, "IndexID mismatch for modify log file "
- "%s", file->filepath);
- ret = -1;
-
- /* we have to rebuild it, make sure it's deleted. */
- (void)unlink(file->filepath);
- }
-
- if (ret > 0 && hdr.sync_id == SYNC_ID_FULL) {
- /* full */
- ret = 0;
- }
-
- if (ret > 0)
- file->fd = fd;
- else
- (void)close(fd);
-
- return ret;
-}
-
-static int modifylog_files_open_or_create(struct mail_modify_log *log)
-{
- int i, ret1, ret2;
-
- for (i = 0; i < 2; i++) {
- ret1 = mail_modifylog_open_and_verify(&log->file1);
- ret2 = mail_modifylog_open_and_verify(&log->file2);
-
- if (ret1 == 1 && ret2 != 1) {
- log->head = log->tail = &log->file1;
- return TRUE;
- }
-
- if (ret1 != 1 && ret2 == 1) {
- log->head = log->tail = &log->file2;
- return TRUE;
- }
-
- if (ret1 == 1 && ret2 == 1) {
- /* both logs were opened ok, which shouldn't happen.
- safest thing to do is to mark both closed,
- delete them and recreate */
- index_set_error(log->index,
- "Index %s has both modify logs open",
- log->index->filepath);
- (void)modifylog_mark_full(&log->file1);
- (void)modifylog_mark_full(&log->file2);
-
- (void)unlink(log->file1.filepath);
- (void)unlink(log->file2.filepath);
-
- modifylog_close_file(&log->file1);
- modifylog_close_file(&log->file2);
- }
-
- ret1 = modifylog_reuse_or_create_file(&log->file1);
- if (ret1 == 1) {
- log->head = log->tail = &log->file1;
- return TRUE;
- }
- if (ret1 == -1)
- break;
-
- /* someone else probably just created the file */
- }
-
- if (ret1 == 0) {
- /* we tried twice */
- index_set_error(log->index, "Couldn't lock modify log file %s",
- log->file1.filepath);
- }
- return FALSE;
-}
-
-static int modifylog_create_anon(struct modify_log_file *file)
-{
- file->mmap_full_length = MODIFY_LOG_INITIAL_SIZE;
- file->mmap_base = mmap_anon(file->mmap_full_length);
- file->header = file->mmap_base;
-
- if (file->mmap_base == MAP_FAILED)
- return modifylog_set_syscall_error(file, "mmap_anon()");
-
- mail_modifylog_init_header(file->log, file->mmap_base);
-
- file->mmap_used_length = file->header->used_file_size;
- file->synced_position = file->mmap_used_length;
-
- file->anon_mmap = TRUE;
- file->filepath = i_strdup_printf("(in-memory modify log for %s)",
- file->log->index->mailbox_path);
- return TRUE;
-}
-
-int mail_modifylog_create(struct mail_index *index)
-{
- struct mail_modify_log *log;
- int ret;
-
- i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
- log = mail_modifylog_new(index);
-
- if (INDEX_IS_IN_MEMORY(index)) {
- if (!modifylog_create_anon(&log->file1)) {
- mail_modifylog_free(log);
- return FALSE;
- }
- } else {
- ret = modifylog_reuse_or_create_file(&log->file1);
- if (ret == 0) {
- index_set_error(log->index,
- "Couldn't lock created modify log file %s",
- log->file1.filepath);
- }
-
- if (ret <= 0 || !mmap_init_update(&log->file1)) {
- /* fatal failure */
- mail_modifylog_free(log);
- return FALSE;
- }
- }
-
- log->head = log->tail = &log->file1;
- return TRUE;
-}
-
-int mail_modifylog_open_or_create(struct mail_index *index)
-{
- struct mail_modify_log *log;
-
- log = mail_modifylog_new(index);
-
- if (!modifylog_files_open_or_create(log) ||
- !mmap_init_update(log->head)) {
- /* fatal failure */
- mail_modifylog_free(log);
- return FALSE;
- }
-
- return TRUE;
-}
-
-void mail_modifylog_free(struct mail_modify_log *log)
-{
- log->index->modifylog = NULL;
-
- modifylog_close_file(&log->file1);
- modifylog_close_file(&log->file2);
-
- i_free(log->file1.filepath);
- i_free(log->file2.filepath);
- i_free(log);
-}
-
-int mail_modifylog_sync_file(struct mail_modify_log *log, int *fsync_fd)
-{
- struct modify_log_file *file = log->head;
-
- *fsync_fd = -1;
-
- if (!file->modified || file->anon_mmap)
- return TRUE;
-
- i_assert(file->mmap_base != NULL);
-
- if (msync(file->mmap_base, file->mmap_used_length, MS_SYNC) < 0)
- return modifylog_set_syscall_error(file, "msync()");
-
- *fsync_fd = file->fd;
- file->modified = FALSE;
- return TRUE;
-}
-
-void mail_modifylog_notify_lock_drop(struct mail_modify_log *log)
-{
- log->head->last_expunge = NULL;
- log->head->last_flags = NULL;
-}
-
-/* if head file is closed, change it */
-static int modifylog_update_head(struct mail_modify_log *log)
-{
- struct modify_log_file *file;
-
- if (!mmap_update(log->head, FALSE))
- return FALSE;
-
- if (log->head->header->sync_id != SYNC_ID_FULL)
- return TRUE;
-
- i_assert(log->head == log->tail);
-
- /* switch file */
- file = log->head == &log->file1 ? &log->file2 : &log->file1;
- if (file->fd == -1) {
- if (mail_modifylog_open_and_verify(file) <= 0) {
- modifylog_set_corrupted(file,
- "Can't switch to open log file");
- return FALSE;
- }
- }
-
- if (!mmap_update(file, TRUE))
- return FALSE;
-
- /* we're non-synced */
- file->synced_id = 0;
- file->synced_position = sizeof(struct modify_log_header);
- log->head = file;
- return TRUE;
-}
-
-static int mmap_update_both(struct mail_modify_log *log)
-{
- if (!modifylog_update_head(log))
- return FALSE;
-
- if (log->head != log->tail) {
- if (!mmap_update(log->tail, FALSE))
- return FALSE;
- }
-
- return TRUE;
-}
-
-static int mail_modifylog_grow(struct modify_log_file *file)
-{
- uoff_t new_fsize;
- void *base;
-
- new_fsize = (uoff_t)file->mmap_full_length + MODIFYLOG_GROW_SIZE;
- i_assert(new_fsize < OFF_T_MAX);
-
- if (file->anon_mmap) {
- i_assert(new_fsize < SSIZE_T_MAX);
-
- base = mremap_anon(file->mmap_base, file->mmap_full_length,
- (size_t)new_fsize, MREMAP_MAYMOVE);
- if (base == MAP_FAILED) {
- modifylog_set_syscall_error(file, "mremap_anon()");
- return FALSE;
- }
-
- file->mmap_base = base;
- file->mmap_full_length = (size_t)new_fsize;
- return TRUE;
- }
-
- if (file_set_size(file->fd, (off_t)new_fsize) < 0)
- return modifylog_set_syscall_error(file, "file_set_size()");
-
- if (!mmap_update(file, TRUE))
- return FALSE;
-
- return TRUE;
-}
-
-static int mail_modifylog_append(struct modify_log_file *file,
- struct modify_log_record **rec,
- int external_change)
-{
- struct modify_log_record *destrec;
-
- i_assert(file->log->index->lock_type == MAIL_LOCK_EXCLUSIVE);
- i_assert(file->header->sync_id != SYNC_ID_FULL);
- i_assert((*rec)->seq1 != 0);
- i_assert((*rec)->uid1 != 0);
-
- if (!external_change) {
- if (file->log->cache_lock_counter !=
- file->log->index->excl_lock_counter) {
- switch (modifylog_have_other_users(file->log, FALSE)) {
- case 0:
- /* we're the only one having this log open,
- no need for modify log. */
- file->log->cache_have_others = FALSE;
- file->log->cache_lock_counter =
- file->log->index->excl_lock_counter;
-
- *rec = NULL;
- return TRUE;
- case -1:
- return FALSE;
- default:
- file->log->cache_have_others = TRUE;
- file->log->cache_lock_counter =
- file->log->index->excl_lock_counter;
- break;
- }
- }
-
- if (!file->log->cache_have_others) {
- *rec = NULL;
- return TRUE;
- }
- }
-
- if (file->mmap_used_length == file->mmap_full_length) {
- if (!mail_modifylog_grow(file))
- return FALSE;
- }
-
- i_assert(file->header->used_file_size == file->mmap_used_length);
- i_assert(file->mmap_used_length + sizeof(struct modify_log_record) <=
- file->mmap_full_length);
-
- destrec = (struct modify_log_record *) ((char *) file->mmap_base +
- file->mmap_used_length);
- memcpy(destrec, *rec, sizeof(struct modify_log_record));
-
- if (!external_change && file->header->sync_id == file->synced_id) {
- file->synced_position += sizeof(struct modify_log_record);
- file->synced_id++;
- }
-
- file->header->used_file_size += sizeof(struct modify_log_record);
- file->mmap_used_length += sizeof(struct modify_log_record);
-
- file->header->sync_id++;
- file->modified = TRUE;
-
- *rec = destrec;
- return TRUE;
-}
-
-int mail_modifylog_add_expunges(struct mail_modify_log *log,
- unsigned int first_seq, unsigned int last_seq,
- unsigned int first_uid, unsigned int last_uid,
- int external_change)
-{
- struct modify_log_file *file;
- struct modify_log_record rec, *recp;
-
- if (!modifylog_update_head(log))
- return FALSE;
-
- file = log->head;
-
- /* expunges must not be added when log isn't synced */
- i_assert(external_change || file->synced_id == file->header->sync_id);
-
- if (file->last_expunge != NULL &&
- file->last_expunge_external == external_change) {
- if (last_seq+1 == file->last_expunge->seq1) {
- i_assert(last_uid < file->last_expunge->uid1);
- file->last_expunge->seq1 = first_seq;
- file->last_expunge->uid1 = first_uid;
- return TRUE;
- } else if (first_seq == file->last_expunge->seq1) {
- /* note that the weird looking logic above is correct.
- it's because of reordered seq numbers. */
- i_assert(first_uid > file->last_expunge->uid2);
- file->last_expunge->seq2 += (last_seq - first_seq) + 1;
- file->last_expunge->uid2 = last_uid;
- return TRUE;
- }
- }
-
- rec.type = RECORD_TYPE_EXPUNGE;
- rec.seq1 = first_seq; rec.seq2 = last_seq;
- rec.uid1 = first_uid; rec.uid2 = last_uid;
-
- recp = &rec;
- if (!mail_modifylog_append(file, &recp, external_change))
- return FALSE;
-
- file->last_expunge_external = external_change;
- file->last_expunge = recp;
- return TRUE;
-}
-
-int mail_modifylog_add_flags(struct mail_modify_log *log, unsigned int seq,
- unsigned int uid, int external_change)
-{
- struct modify_log_file *file;
- struct modify_log_record rec, *recp;
-
- if (!modifylog_update_head(log))
- return FALSE;
-
- file = log->head;
-
- if (file->last_flags != NULL &&
- file->last_flags_external == external_change) {
- if (seq+1 == file->last_flags->seq1) {
- file->last_flags->seq1 = seq;
- file->last_flags->uid1 = uid;
- return TRUE;
- } else if (seq-1 == file->last_flags->seq2) {
- file->last_flags->seq2 = seq;
- file->last_flags->uid2 = uid;
- return TRUE;
- }
- }
-
- rec.type = RECORD_TYPE_FLAGS_CHANGED;
- rec.seq1 = rec.seq2 = seq;
- rec.uid1 = rec.uid2 = uid;
-
- recp = &rec;
- if (!mail_modifylog_append(file, &recp, external_change))
- return FALSE;
-
- file->last_flags_external = external_change;
- file->last_flags = recp;
- return TRUE;
-}
-
-static void
-mail_modifylog_get_nonsynced_file(struct modify_log_file *file,
- const struct modify_log_record **arr,
- unsigned int *count)
-{
- struct modify_log_record *end_rec;
-
- i_assert(file->synced_position <= file->mmap_used_length);
- i_assert(file->synced_position >= sizeof(struct modify_log_header));
-
- *arr = (struct modify_log_record *) ((char *) file->mmap_base +
- file->synced_position);
- end_rec = (struct modify_log_record *) ((char *) file->mmap_base +
- file->mmap_used_length);
- *count = (unsigned int) (end_rec - *arr);
-}
-
-int mail_modifylog_get_nonsynced(struct mail_modify_log *log,
- const struct modify_log_record **arr1,
- unsigned int *count1,
- const struct modify_log_record **arr2,
- unsigned int *count2)
-{
- i_assert(log->index->lock_type != MAIL_LOCK_UNLOCK);
-
- *arr1 = *arr2 = NULL;
- *count1 = *count2 = 0;
-
- if (!mmap_update_both(log))
- return FALSE;
-
- mail_modifylog_get_nonsynced_file(log->tail, arr1, count1);
- if (log->head != log->tail)
- mail_modifylog_get_nonsynced_file(log->head, arr2, count2);
-
- return TRUE;
-}
-
-static int mail_modifylog_try_truncate(struct modify_log_file *file)
-{
- if (modifylog_have_other_users(file->log, TRUE) != 0)
- return FALSE;
-
-#ifdef DEBUG
- mprotect(file->mmap_base, sizeof(struct modify_log_header),
- PROT_READ | PROT_WRITE);
-#endif
- file->header->sync_id = 0;
- file->header->used_file_size = sizeof(struct modify_log_header);
-
- if (msync(file->mmap_base,
- sizeof(struct modify_log_header), MS_SYNC) < 0) {
- modifylog_set_syscall_error(file, "msync()");
- return FALSE;
- }
-
- file->synced_id = 0;
- file->synced_position = sizeof(struct modify_log_header);
-
- if (file_set_size(file->fd, MODIFY_LOG_INITIAL_SIZE) < 0)
- modifylog_set_syscall_error(file, "file_set_size()");
-
- return TRUE;
-}
-
-/* switches to active modify log, updating our sync mark to end of it */
-static int mail_modifylog_switch_file(struct mail_modify_log *log)
-{
- struct modify_log_file *file;
-
- (void)mail_modifylog_try_truncate(log->tail);
-
- file = log->tail == &log->file1 ? &log->file2 : &log->file1;
- if (file->fd == -1) {
- if (mail_modifylog_open_and_verify(file) <= 0) {
- modifylog_set_corrupted(file,
- "Can't switch to open log file");
- return FALSE;
- }
- }
-
- modifylog_munmap(log->tail);
-
- log->head = log->tail = file;
- return mmap_init_update(log->head);
-}
-
-static int mail_modifylog_try_switch_file(struct mail_modify_log *log)
-{
- struct modify_log_file *file;
-
- if (log->head->anon_mmap)
- return TRUE;
-
- if (mail_modifylog_try_truncate(log->tail)) {
- /* no need to switch, we're the only user and we just
- truncated it */
- return TRUE;
- }
-
- file = log->head == &log->file1 ? &log->file2 : &log->file1;
- if (modifylog_reuse_or_create_file(file) != 1) {
- /* locked or error, keep using the old log */
- return TRUE;
- }
-
- if (!modifylog_mark_full(log->head))
- return FALSE;
-
- modifylog_munmap(log->head);
-
- log->head = log->tail = file;
- return mmap_init_update(log->head);
-}
-
-int mail_modifylog_mark_synced(struct mail_modify_log *log)
-{
- i_assert(log->index->lock_type != MAIL_LOCK_UNLOCK);
-
- if (!mmap_update_both(log))
- return FALSE;
-
- if (log->tail->header->sync_id == SYNC_ID_FULL) {
- /* tail file is full, switch to next one */
- return mail_modifylog_switch_file(log);
- }
-
- log->tail = log->head;
- if (log->head->synced_id != log->head->header->sync_id) {
- log->head->synced_id = log->head->header->sync_id;
- log->head->synced_position = log->head->mmap_used_length;
- }
-
- if (log->head->mmap_used_length > MAX_MODIFYLOG_SIZE) {
- /* if the other file isn't locked, switch to it */
- return mail_modifylog_try_switch_file(log);
- }
-
- return TRUE;
-}
-
-static int compare_expunge(const void *p1, const void *p2)
-{
- const struct modify_log_expunge *e1 = p1;
- const struct modify_log_expunge *e2 = p2;
-
- return e1->uid1 < e2->uid1 ? -1 : e1->uid1 > e2->uid1 ? 1 : 0;
-}
-
-static struct modify_log_record *modifylog_first(struct mail_modify_log *log)
-{
- struct modify_log_file *file;
- struct modify_log_record *rec;
-
- file = log->tail;
- rec = (struct modify_log_record *) ((char *) file->mmap_base +
- file->synced_position);
- log->iterator_end = (struct modify_log_record *)
- ((char *) file->mmap_base + file->mmap_used_length);
- return rec < log->iterator_end ? rec : NULL;
-}
-
-static struct modify_log_record *
-modifylog_next(struct mail_modify_log *log, struct modify_log_record *rec)
-{
- struct modify_log_file *file;
-
- rec++;
- if (rec < log->iterator_end)
- return rec;
-
- file = log->head;
- if ((char *) rec == (char *) file->mmap_base + file->mmap_used_length)
- return NULL; /* end of head */
-
- /* end of tail, jump to beginning of head */
- rec = (struct modify_log_record *) ((char *) file->mmap_base +
- sizeof(struct modify_log_header));
- log->iterator_end = (struct modify_log_record *)
- ((char *) file->mmap_base + file->mmap_used_length);
- return rec < log->iterator_end ? rec : NULL;
-}
-
-static unsigned int
-modifylog_get_record_count_after(struct mail_modify_log *log,
- struct modify_log_record *rec)
-{
- unsigned int count = 0;
-
- if (log->head == log->tail ||
- IS_PTR_IN_RANGE(rec, log->head->mmap_base,
- log->head->mmap_used_length)) {
- /* only head */
- count = (log->head->mmap_used_length -
- MODIFYLOG_FILE_POSITION(log->head, rec)) /
- sizeof(struct modify_log_record);
- } else {
- /* tail */
- count = (log->tail->mmap_used_length -
- MODIFYLOG_FILE_POSITION(log->tail, rec)) /
- sizeof(struct modify_log_record);
-
- if (log->head != log->tail) {
- /* + head */
- count += (log->tail->mmap_used_length -
- sizeof(struct modify_log_header)) /
- sizeof(struct modify_log_record);
- }
- }
-
- return count;
-}
-
-const struct modify_log_expunge *
-mail_modifylog_seq_get_expunges(struct mail_modify_log *log,
- unsigned int first_seq,
- unsigned int last_seq,
- unsigned int *expunges_before)
-{
- struct modify_log_record *rec;
- struct modify_log_expunge expunge, *expunges;
- buffer_t *buf;
- size_t count;
- unsigned int before, max_records;
-
- i_assert(log->index->lock_type != MAIL_LOCK_UNLOCK);
-
- *expunges_before = 0;
-
- if (!mmap_update_both(log))
- return NULL;
-
- /* find the first expunged message that affects our range */
- rec = modifylog_first(log);
- while (rec != NULL) {
- if (rec->type == RECORD_TYPE_EXPUNGE && rec->seq1 <= last_seq)
- break;
-
- rec = modifylog_next(log, rec);
- }
-
- if (rec == NULL) {
- /* none found */
- return &no_expunges;
- }
-
- /* allocate memory for the returned array. the file size - synced
- position should be quite near the amount of memory we need, unless
- there's lots of FLAGS_CHANGED records which is why there's the
- second check to make sure it's not unneededly large. */
- max_records = modifylog_get_record_count_after(log, rec);
- if (max_records > last_seq - first_seq + 1)
- max_records = last_seq - first_seq + 1;
-
- i_assert((max_records+1) <
- SSIZE_T_MAX / sizeof(struct modify_log_expunge));
- buf = buffer_create_static_hard(pool_datastack_create(),
- (max_records+1) *
- sizeof(struct modify_log_expunge));
-
- before = 0;
- for (; rec != NULL; rec = modifylog_next(log, rec)) {
- if (rec->type != RECORD_TYPE_EXPUNGE)
- continue;
-
- if (rec->seq2 < first_seq) {
- /* before our range */
- before += rec->seq2 - rec->seq1 + 1;
- } else if (rec->seq1 <= last_seq && rec->seq2 >= first_seq) {
- /* within our range, at least partially */
- if (max_records-- == 0) {
- /* log contains more data than it should
- have - must be corrupted. */
- modifylog_set_corrupted(log->tail,
- "Contains more data than expected");
- return NULL;
- }
-
- if (rec->seq1 < first_seq) {
- /* partial initial match, update
- before-counter */
- before += first_seq - rec->seq1;
- expunge.seq_count = rec->seq2 - first_seq + 1;
- } else {
- expunge.seq_count = rec->seq2 - rec->seq1 + 1;
- }
-
- expunge.uid1 = rec->uid1;
- expunge.uid2 = rec->uid2;
- buffer_append(buf, &expunge, sizeof(expunge));
- }
-
- if (rec->seq1 <= last_seq) {
- /* update the seq. numbers so they can be compared */
- last_seq -= I_MIN(rec->seq2, last_seq) -
- rec->seq1 + 1;
-
- if (rec->seq1 < first_seq) {
- first_seq -= I_MIN(rec->seq2, first_seq-1) -
- rec->seq1 + 1;
- }
- }
- }
-
- /* terminate the array */
- buffer_set_used_size(buf, buffer_get_used_size(buf) + sizeof(expunge));
-
- /* extract the array from buffer */
- count = buffer_get_used_size(buf) / sizeof(expunge);
- expunges = buffer_free_without_data(buf);
-
- /* sort the UID array, not including the terminating 0 */
- qsort(expunges, count-1, sizeof(expunge), compare_expunge);
-
- *expunges_before = before;
- return expunges;
-}
-
-const struct modify_log_expunge *
-mail_modifylog_uid_get_expunges(struct mail_modify_log *log,
- unsigned int first_uid,
- unsigned int last_uid,
- unsigned int *expunges_before)
-{
- /* pretty much copy&pasted from sequence code above ..
- kind of annoying */
- struct modify_log_record *rec;
- struct modify_log_expunge expunge, *expunges;
- buffer_t *buf;
- size_t count;
- unsigned int before, max_records;
-
- i_assert(log->index->lock_type != MAIL_LOCK_UNLOCK);
-
- *expunges_before = 0;
-
- if (!mmap_update_both(log))
- return NULL;
-
- /* find the first expunged message that affects our range */
- rec = modifylog_first(log);
- while (rec != NULL) {
- if (rec->type == RECORD_TYPE_EXPUNGE && rec->uid1 <= last_uid)
- break;
-
- rec = modifylog_next(log, rec);
- }
-
- if (rec == NULL) {
- /* none found */
- return &no_expunges;
- }
-
- /* allocate memory for the returned array. the file size - synced
- position should be quite near the amount of memory we need, unless
- there's lots of FLAGS_CHANGED records which is why there's the
- second check to make sure it's not unneededly large. */
- max_records = modifylog_get_record_count_after(log, rec);
- if (max_records > last_uid - first_uid + 1)
- max_records = last_uid - first_uid + 1;
-
- i_assert((max_records+1) <
- SSIZE_T_MAX / sizeof(struct modify_log_expunge));
- buf = buffer_create_static_hard(pool_datastack_create(),
- (max_records+1) *
- sizeof(struct modify_log_expunge));
-
- before = 0;
- for (; rec != NULL; rec = modifylog_next(log, rec)) {
- if (rec->type != RECORD_TYPE_EXPUNGE)
- continue;
-
- if (rec->uid2 < first_uid) {
- /* before our range */
- before += rec->seq2 - rec->seq1 + 1;
- } else if (rec->uid1 <= last_uid && rec->uid2 >= first_uid) {
- /* within our range, at least partially */
- if (max_records-- == 0) {
- /* log contains more data than it should
- have - must be corrupted. */
- modifylog_set_corrupted(log->tail,
- "Contains more data than expected");
- return NULL;
- }
-
- expunge.uid1 = rec->uid1;
- expunge.uid2 = rec->uid2;
- expunge.seq_count = rec->seq2 -rec->seq1 + 1;
- buffer_append(buf, &expunge, sizeof(expunge));
- }
- }
-
- /* terminate the array */
- buffer_set_used_size(buf, buffer_get_used_size(buf) + sizeof(expunge));
-
- /* extract the array from buffer */
- count = buffer_get_used_size(buf) / sizeof(expunge);
- expunges = buffer_free_without_data(buf);
-
- /* sort the UID array, not including the terminating 0 */
- qsort(expunges, count-1, sizeof(expunge), compare_expunge);
-
- *expunges_before = before;
- return expunges;
-}
-
-static unsigned int
-modifylog_file_get_expunge_count(struct modify_log_file *file)
-{
- struct modify_log_record *rec, *end_rec;
- unsigned int expunges;
-
- /* find the first expunged message that affects our range */
- rec = (struct modify_log_record *) ((char *) file->mmap_base +
- file->synced_position);
- end_rec = (struct modify_log_record *) ((char *) file->mmap_base +
- file->mmap_used_length);
-
- expunges = 0;
- while (rec < end_rec) {
- if (rec->type == RECORD_TYPE_EXPUNGE)
- expunges += rec->seq2 - rec->seq1 + 1;
- rec++;
- }
-
- return expunges;
-}
-
-unsigned int mail_modifylog_get_expunge_count(struct mail_modify_log *log)
-{
- unsigned int expunges;
-
- i_assert(log->index->lock_type != MAIL_LOCK_UNLOCK);
-
- if (!mmap_update_both(log))
- return 0;
-
- expunges = modifylog_file_get_expunge_count(log->tail);
- if (log->tail != log->head)
- expunges += modifylog_file_get_expunge_count(log->head);
-
- return expunges;
-}
+++ /dev/null
-#ifndef __MAIL_MODIFYLOG_H
-#define __MAIL_MODIFYLOG_H
-
-enum modify_log_record_type {
- RECORD_TYPE_EXPUNGE,
- RECORD_TYPE_FLAGS_CHANGED
-};
-
-/* if sync_id has this value, the log file is full and should be
- deleted or reused. */
-#define SYNC_ID_FULL ((unsigned int)-1)
-
-struct modify_log_header {
- unsigned int indexid;
- unsigned int sync_id;
- uoff_t used_file_size;
-};
-
-struct modify_log_record {
- unsigned int type;
- unsigned int seq1, seq2;
- unsigned int uid1, uid2;
-};
-
-/* for mail_modifylog_*_get_expunges() */
-struct modify_log_expunge {
- unsigned int uid1, uid2; /* NOTE: may be outside wanted range */
- unsigned int seq_count;
-};
-
-/* NOTE: All these functions require the index file to be locked. */
-
-int mail_modifylog_create(struct mail_index *index);
-int mail_modifylog_open_or_create(struct mail_index *index);
-void mail_modifylog_free(struct mail_modify_log *log);
-
-/* Append EXPUGE or FLAGS entry to modify log. Index must be exclusively
- locked before calling these functions, and modifylog must have been
- marked synced within the same lock. */
-int mail_modifylog_add_expunges(struct mail_modify_log *log,
- unsigned int first_seq, unsigned int last_seq,
- unsigned int first_uid, unsigned int last_uid,
- int external_change);
-int mail_modifylog_add_flags(struct mail_modify_log *log, unsigned int seq,
- unsigned int uid, int external_change);
-
-/* Synchronize the data into disk */
-int mail_modifylog_sync_file(struct mail_modify_log *log, int *fsync_fd);
-
-/* Must be called when exclusive lock is dropped from index. */
-void mail_modifylog_notify_lock_drop(struct mail_modify_log *log);
-
-/* Updates arr and count parameters to list nonsynced log entries.
- Returns TRUE if successful. */
-int mail_modifylog_get_nonsynced(struct mail_modify_log *log,
- const struct modify_log_record **arr1,
- unsigned int *count1,
- const struct modify_log_record **arr2,
- unsigned int *count2);
-
-/* Marks the modify log as being synced with in-memory state. */
-int mail_modifylog_mark_synced(struct mail_modify_log *log);
-
-/* Finds expunged messages for the given sequence range, and number of
- expunged messages before the range. Returns 0,0 terminated list of
- expunged UIDs, or NULL if error occured.
-
- Note that returned UID range may not be exact for first returned
- expunge record. For example fetching range 9:10 may return
- expunges_before=8, {uid1=1, uid2=9, seq_count=1} if only message 10
- exists.
-
- Also the last expunge record's both uid and seq_count ranges may go
- past last_seq */
-const struct modify_log_expunge *
-mail_modifylog_seq_get_expunges(struct mail_modify_log *log,
- unsigned int first_seq,
- unsigned int last_seq,
- unsigned int *expunges_before);
-
-/* Like above, but for given UID range. expunges_before is treated a bit
- differently however. It specifies the number of messages deleted before
- the first returned expunge-record, which may partially be before our
- wanted range. For example fetching range 9:10 may return
- expunges_before=0, {uid1=1, uid2=9, seq_count=9} if only message 10
- exists. This is because we have no idea how many messages there are
- between UIDs since they're not guaranteed to be contiguous. */
-const struct modify_log_expunge *
-mail_modifylog_uid_get_expunges(struct mail_modify_log *log,
- unsigned int first_uid,
- unsigned int last_uid,
- unsigned int *expunges_before);
-
-/* Get number of non-synced expunges in modify log. */
-unsigned int mail_modifylog_get_expunge_count(struct mail_modify_log *log);
-
-#endif
+++ /dev/null
-/* Copyright (C) 2002-2003 Timo Sirainen */
-
-#include "lib.h"
-#include "maildir-index.h"
-#include "mail-cache.h"
-
-int maildir_cache_update_file(struct mail_cache_transaction_ctx **trans_ctx,
- struct mail_index *index,
- struct mail_index_record *rec, const char *fname,
- int new_dir)
-{
- enum mail_cache_field cached_fields;
- enum mail_index_record_flag index_flags;
- uoff_t virtual_size;
- const char *p;
-
- if (*trans_ctx == NULL) {
- if (mail_cache_transaction_begin(index->cache,
- TRUE, trans_ctx) <= 0)
- return FALSE;
- }
-
- cached_fields = mail_cache_get_fields(index->cache, rec);
- if ((cached_fields & MAIL_CACHE_INDEX_FLAGS) == 0) {
- /* always set index flags */
- index_flags = new_dir ? MAIL_INDEX_FLAG_MAILDIR_NEW : 0;
- if (!mail_cache_add(*trans_ctx, rec, MAIL_CACHE_INDEX_FLAGS,
- &index_flags, sizeof(index_flags)))
- return FALSE;
- }
-
- /* set virtual size if found from file name */
- p = strstr(fname, ",W=");
- if (p != NULL && (cached_fields & MAIL_CACHE_VIRTUAL_FULL_SIZE) == 0) {
- p += 3;
- virtual_size = 0;
- while (*p >= '0' && *p <= '9') {
- virtual_size = virtual_size * 10 + (*p - '0');
- p++;
- }
-
- if (*p == ':' || *p == ',' || *p != '\0') {
- if (!mail_cache_add(*trans_ctx, rec,
- MAIL_CACHE_VIRTUAL_FULL_SIZE,
- &virtual_size,
- sizeof(virtual_size)))
- return FALSE;
- }
- }
-
- if ((cached_fields & MAIL_CACHE_LOCATION) == 0) {
- /* always set location */
- if (!mail_cache_add(*trans_ctx, rec, MAIL_CACHE_LOCATION,
- fname, strlen(fname)+1))
- return FALSE;
- }
-
- return TRUE;
-}
-
-int maildir_index_append_file(struct mail_cache_transaction_ctx **trans_ctx,
- struct mail_index *index, const char *fname,
- int new_dir)
-{
- struct mail_index_record *rec;
-
- rec = index->append(index);
- if (rec == NULL)
- return FALSE;
-
- /* set message flags from file name */
- rec->msg_flags = maildir_filename_get_flags(fname, 0);
- mail_index_mark_flag_changes(index, rec, 0, rec->msg_flags);
-
- return maildir_cache_update_file(trans_ctx, index, rec, fname, new_dir);
-}
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "ioloop.h"
-#include "maildir-index.h"
-
-#include <dirent.h>
-#include <unistd.h>
-#include <sys/stat.h>
-
-/* Clean files from tmp/ if they're older than 36 hours */
-#define MAILDIR_CLEANUP_TIME (60 * 60 * 36)
-
-void maildir_clean_tmp(const char *dir)
-{
- time_t cleanup_time = ioloop_time - MAILDIR_CLEANUP_TIME;
- DIR *dirp;
- struct dirent *d;
- struct stat st;
- const char *path;
-
- dirp = opendir(dir);
- if (dirp == NULL) {
- i_error("opendir(%s) failed: %m", dir);
- return;
- }
-
- while ((d = readdir(dirp)) != NULL) {
- if (strcmp(d->d_name, ".") == 0 ||
- strcmp(d->d_name, "..") == 0)
- continue;
-
- t_push();
- path = t_strconcat(dir, "/", d->d_name, NULL);
- if (stat(path, &st) < 0) {
- if (errno != ENOENT)
- i_error("stat(%s) failed: %m", path);
- } else if (st.st_mtime < cleanup_time &&
- st.st_atime < cleanup_time &&
- !S_ISDIR(st.st_mode)) {
- if (unlink(path) < 0 && errno != ENOENT)
- i_error("unlink(%s) failed: %m", path);
- }
- t_pop();
- }
-
- if (closedir(dirp) < 0)
- i_error("closedir(%s) failed: %m", dir);
-}
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "ioloop.h"
-#include "maildir-index.h"
-#include "mail-index-util.h"
-
-#include <unistd.h>
-
-static int do_expunge(struct mail_index *index, const char *path, void *context)
-{
- int *found = context;
-
- if (unlink(path) < 0) {
- if (errno == ENOENT)
- return 0;
- if (errno == EACCES) {
- index->mailbox_readonly = TRUE;
- return 1;
- }
-
- index_set_error(index, "unlink(%s) failed: %m", path);
- return -1;
- }
-
- *found = TRUE;
- return 1;
-}
-
-int maildir_expunge_mail(struct mail_index *index,
- struct mail_index_record *rec)
-{
- int found = FALSE;
-
- if (!maildir_file_do(index, rec, do_expunge, &found))
- return FALSE;
-
- if (found) {
- /* if we're in out-of-space condition, reset it since we'll
- probably have enough space now. */
- index->maildir_keep_new = FALSE;
- if (index->next_dirty_flags_flush != 0)
- index->next_dirty_flags_flush = ioloop_time;
-
- /* cur/ was updated, set it dirty-synced */
- index->sync_dirty_stamp = ioloop_time;
- index->sync_stamp = ioloop_time;
- }
- return TRUE;
-}
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "ioloop.h"
-#include "hash.h"
-#include "hostpid.h"
-#include "str.h"
-#include "maildir-index.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-
-#include <stdio.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <time.h>
-
-extern struct mail_index maildir_index;
-
-static int maildir_index_open(struct mail_index *index,
- enum mail_index_open_flags flags)
-{
- maildir_clean_tmp(t_strconcat(index->mailbox_path, "/tmp", NULL));
- return mail_index_open(index, flags);
-}
-
-const char *maildir_get_location(struct mail_index *index,
- struct mail_index_record *rec, int *new_dir)
-{
- const char *fname, *new_fname;
-
- if (new_dir != NULL)
- *new_dir = FALSE;
-
- if (index->new_filenames != NULL) {
- /* this has the most up-to-date filename */
- new_fname = hash_lookup(index->new_filenames,
- POINTER_CAST(rec->uid));
- if (new_fname != NULL) {
- if (*new_fname == '/') {
- new_fname++;
- if (new_dir != NULL)
- *new_dir = TRUE;
- }
- return new_fname;
- }
- }
-
- /* cache file file should give us at least the base name. */
- fname = mail_cache_lookup_string_field(index->cache, rec,
- MAIL_CACHE_LOCATION);
- if (fname == NULL) {
- /* Not cached, we'll have to resync the directory. */
- return NULL;
- }
-
- if (new_dir != NULL) {
- *new_dir = (mail_cache_get_index_flags(index->cache, rec) &
- MAIL_INDEX_FLAG_MAILDIR_NEW) != 0;
- }
-
- return fname;
-}
-
-static int
-maildir_file_do_try(struct mail_index *index, struct mail_index_record *rec,
- const char **fname,
- maildir_file_do_func *func, void *context)
-{
- const char *path;
- int ret, new_dir;
-
- *fname = maildir_get_location(index, rec, &new_dir);
- if (*fname == NULL)
- return 0;
-
- if (new_dir) {
- /* probably in new/ dir */
- path = t_strconcat(index->mailbox_path, "/new/", *fname, NULL);
- ret = func(index, path, context);
- if (ret != 0)
- return ret;
- }
-
- path = t_strconcat(index->mailbox_path, "/cur/", *fname, NULL);
- return func(index, path, context);
-}
-
-int maildir_file_do(struct mail_index *index, struct mail_index_record *rec,
- maildir_file_do_func *func, void *context)
-{
- const char *fname;
- int i, ret, found;
-
- ret = maildir_file_do_try(index, rec, &fname, func, context);
- for (i = 0; i < 10 && ret == 0; i++) {
- /* file is either renamed or deleted. sync the maildir and
- see which one. if file appears to be renamed constantly,
- don't try to open it more than 10 times. */
- fname = t_strdup(fname);
- if (!maildir_index_sync_readonly(index, fname, &found))
- return FALSE;
-
- if (!found && fname != NULL)
- return TRUE;
-
- ret = maildir_file_do_try(index, rec, &fname, func, context);
- }
-
- return ret >= 0;
-}
-
-const char *maildir_generate_tmp_filename(const struct timeval *tv)
-{
- static unsigned int create_count = 0;
- static time_t first_stamp = 0;
-
- if (first_stamp == 0 || first_stamp == ioloop_time) {
- /* it's possible that within last second another process had
- the same UID as us. Use usecs to make sure we don't create
- duplicate base name. */
- first_stamp = ioloop_time;
- return t_strdup_printf("%s.P%sQ%uM%s.%s",
- dec2str(tv->tv_sec), my_pid,
- create_count++,
- dec2str(tv->tv_usec), my_hostname);
- } else {
- /* Don't bother with usecs. Saves a bit space :) */
- return t_strdup_printf("%s.P%sQ%u.%s",
- dec2str(tv->tv_sec), my_pid,
- create_count++, my_hostname);
- }
-}
-
-int maildir_create_tmp(struct mail_index *index, const char *dir, mode_t mode,
- const char **fname)
-{
- const char *path, *tmp_fname;
- struct stat st;
- struct timeval *tv, tv_now;
- pool_t pool;
- int fd;
-
- tv = &ioloop_timeval;
- pool = pool_alloconly_create("maildir_tmp", 4096);
- for (;;) {
- p_clear(pool);
- tmp_fname = maildir_generate_tmp_filename(tv);
-
- path = p_strconcat(pool, dir, "/", tmp_fname, NULL);
- if (stat(path, &st) < 0 && errno == ENOENT) {
- /* doesn't exist */
- mode_t old_mask = umask(0);
- fd = open(path, O_WRONLY | O_CREAT | O_EXCL, mode);
- umask(old_mask);
- if (fd != -1 || errno != EEXIST)
- break;
- }
-
- /* wait and try again - very unlikely */
- sleep(2);
- tv = &tv_now;
- if (gettimeofday(&tv_now, NULL) < 0)
- i_fatal("gettimeofday(): %m");
- }
-
- *fname = t_strdup(path);
- if (fd == -1)
- index_file_set_syscall_error(index, path, "open()");
-
- pool_unref(pool);
- return fd;
-}
-
-enum mail_flags maildir_filename_get_flags(const char *fname,
- enum mail_flags default_flags)
-{
- const char *info;
- enum mail_flags flags;
-
- info = strchr(fname, ':');
- if (info == NULL || info[1] != '2' || info[2] != ',')
- return default_flags;
-
- flags = 0;
- for (info += 3; *info != '\0' && *info != ','; info++) {
- switch (*info) {
- case 'R': /* replied */
- flags |= MAIL_ANSWERED;
- break;
- case 'S': /* seen */
- flags |= MAIL_SEEN;
- break;
- case 'T': /* trashed */
- flags |= MAIL_DELETED;
- break;
- case 'D': /* draft */
- flags |= MAIL_DRAFT;
- break;
- case 'F': /* flagged */
- flags |= MAIL_FLAGGED;
- break;
- default:
- if (*info >= 'a' && *info <= 'z') {
- /* custom flag */
- flags |= 1 << (MAIL_CUSTOM_FLAG_1_BIT +
- *info-'a');
- break;
- }
-
- /* unknown flag - ignore */
- break;
- }
- }
-
- return flags;
-}
-
-const char *maildir_filename_set_flags(const char *fname, enum mail_flags flags)
-{
- string_t *flags_str;
- const char *info, *oldflags;
- int i, nextflag;
-
- /* remove the old :info from file name, and get the old flags */
- info = strrchr(fname, ':');
- if (info != NULL && strrchr(fname, '/') > info)
- info = NULL;
-
- oldflags = "";
- if (info != NULL) {
- fname = t_strdup_until(fname, info);
- if (info[1] == '2' && info[2] == ',')
- oldflags = info+3;
- }
-
- /* insert the new flags between old flags. flags must be sorted by
- their ASCII code. unknown flags are kept. */
- flags_str = t_str_new(256);
- str_append(flags_str, fname);
- str_append(flags_str, ":2,");
- for (;;) {
- /* skip all known flags */
- while (*oldflags == 'D' || *oldflags == 'F' ||
- *oldflags == 'R' || *oldflags == 'S' ||
- *oldflags == 'T' ||
- (*oldflags >= 'a' && *oldflags <= 'z'))
- oldflags++;
-
- nextflag = *oldflags == '\0' || *oldflags == ',' ? 256 :
- (unsigned char) *oldflags;
-
- if ((flags & MAIL_DRAFT) && nextflag > 'D') {
- str_append_c(flags_str, 'D');
- flags &= ~MAIL_DRAFT;
- }
- if ((flags & MAIL_FLAGGED) && nextflag > 'F') {
- str_append_c(flags_str, 'F');
- flags &= ~MAIL_FLAGGED;
- }
- if ((flags & MAIL_ANSWERED) && nextflag > 'R') {
- str_append_c(flags_str, 'R');
- flags &= ~MAIL_ANSWERED;
- }
- if ((flags & MAIL_SEEN) && nextflag > 'S') {
- str_append_c(flags_str, 'S');
- flags &= ~MAIL_SEEN;
- }
- if ((flags & MAIL_DELETED) && nextflag > 'T') {
- str_append_c(flags_str, 'T');
- flags &= ~MAIL_DELETED;
- }
-
- if ((flags & MAIL_CUSTOM_FLAGS_MASK) && nextflag > 'a') {
- for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
- if (flags & (1 << (i + MAIL_CUSTOM_FLAG_1_BIT)))
- str_append_c(flags_str, 'a' + i);
- }
- flags &= ~MAIL_CUSTOM_FLAGS_MASK;
- }
-
- if (*oldflags == '\0' || *oldflags == ',')
- break;
-
- str_append_c(flags_str, *oldflags);
- oldflags++;
- }
-
- if (*oldflags == ',') {
- /* another flagset, we don't know about these, just keep them */
- while (*oldflags != '\0')
- str_append_c(flags_str, *oldflags++);
- }
-
- return str_c(flags_str);
-}
-
-void maildir_index_update_filename(struct mail_index *index, unsigned int uid,
- const char *fname, int new_dir)
-{
- const char *new_fname, *old_fname;
-
- if (index->new_filename_pool == NULL) {
- index->new_filename_pool =
- pool_alloconly_create("Maildir filenames", 10240);
- }
- if (index->new_filenames == NULL) {
- index->new_filenames =
- hash_create(system_pool, index->new_filename_pool, 0,
- NULL, NULL);
- }
-
- t_push();
- new_fname = !new_dir ? fname : t_strconcat("/", fname, NULL);
- old_fname = hash_lookup(index->new_filenames, POINTER_CAST(uid));
- if (old_fname == NULL || strcmp(old_fname, new_fname) != 0) {
- hash_insert(index->new_filenames, POINTER_CAST(uid),
- p_strdup(index->new_filename_pool, new_fname));
- }
- t_pop();
-}
-
-struct mail_index *
-maildir_index_alloc(const char *maildir, const char *index_dir,
- const char *control_dir)
-{
- struct mail_index *index;
-
- i_assert(maildir != NULL);
- i_assert(control_dir != NULL);
-
- index = i_new(struct mail_index, 1);
- memcpy(index, &maildir_index, sizeof(struct mail_index));
-
- index->maildir_lock_fd = -1;
- index->mailbox_path = i_strdup(maildir);
- index->control_dir = i_strdup(control_dir);
- index->mailbox_readonly = access(maildir, W_OK) < 0;
- mail_index_init(index, index_dir);
- return index;
-}
-
-static void maildir_index_free(struct mail_index *index)
-{
- if (index->new_filenames != NULL)
- hash_destroy(index->new_filenames);
- if (index->new_filename_pool != NULL)
- pool_unref(index->new_filename_pool);
-
- mail_index_close(index);
- i_free(index->dir);
- i_free(index->mailbox_path);
- i_free(index->control_dir);
- i_free(index);
-}
-
-static int do_get_received_date(struct mail_index *index,
- const char *path, void *context)
-{
- time_t *date = context;
- struct stat st;
-
- if (stat(path, &st) < 0) {
- if (errno == ENOENT)
- return 0;
- index_file_set_syscall_error(index, path, "stat()");
- return -1;
- }
-
- *date = st.st_mtime;
- return 1;
-}
-
-static time_t maildir_get_received_date(struct mail_index *index,
- struct mail_index_record *rec)
-{
- time_t date;
-
- /* try getting it from cache */
- if (mail_cache_copy_fixed_field(index->cache, rec,
- MAIL_CACHE_RECEIVED_DATE,
- &date, sizeof(date)))
- return date;
-
- date = (time_t)-1;
- if (!maildir_file_do(index, rec, do_get_received_date, &date))
- return (time_t)-1;
-
- return date;
-}
-
-struct mail_index maildir_index = {
- maildir_index_open,
- maildir_index_free,
- mail_index_set_lock,
- mail_index_try_lock,
- mail_index_set_lock_notify_callback,
- mail_index_rebuild,
- mail_index_fsck,
- maildir_index_sync,
- mail_index_get_header,
- mail_index_lookup,
- mail_index_next,
- mail_index_lookup_uid_range,
- maildir_open_mail,
- maildir_get_received_date,
- mail_index_expunge,
- maildir_index_update_flags,
- mail_index_append,
- mail_index_get_last_error,
- mail_index_get_last_error_text,
-
- MAIL_INDEX_PRIVATE_FILL
-};
+++ /dev/null
-#ifndef __MAILDIR_INDEX_H
-#define __MAILDIR_INDEX_H
-
-struct mail_cache_transaction_ctx;
-
-#include <sys/time.h>
-#include "mail-index.h"
-
-/* How often to try to flush dirty flags. */
-#define MAILDIR_DIRTY_FLUSH_TIMEOUT (60*5)
-
-/* Return -1 = error, 0 = file not found, 1 = ok */
-typedef int maildir_file_do_func(struct mail_index *index,
- const char *path, void *context);
-
-struct mail_index *
-maildir_index_alloc(const char *maildir, const char *index_dir,
- const char *control_dir);
-
-/* Return new filename base to save into tmp/ */
-const char *maildir_generate_tmp_filename(const struct timeval *tv);
-int maildir_create_tmp(struct mail_index *index, const char *dir, mode_t mode,
- const char **path);
-
-const char *maildir_get_location(struct mail_index *index,
- struct mail_index_record *rec, int *new_dir);
-int maildir_file_do(struct mail_index *index, struct mail_index_record *rec,
- maildir_file_do_func *func, void *context);
-enum mail_flags maildir_filename_get_flags(const char *fname,
- enum mail_flags default_flags);
-const char *maildir_filename_set_flags(const char *fname,
- enum mail_flags flags);
-void maildir_index_update_filename(struct mail_index *index, unsigned int uid,
- const char *fname, int new_dir);
-
-int maildir_index_sync_readonly(struct mail_index *index,
- const char *fname, int *found);
-int maildir_index_sync(struct mail_index *index, int minimal_sync,
- enum mail_lock_type lock_type, int *changes);
-
-int maildir_cache_update_file(struct mail_cache_transaction_ctx **trans_ctx,
- struct mail_index *index,
- struct mail_index_record *rec, const char *fname,
- int new_dir);
-int maildir_index_append_file(struct mail_cache_transaction_ctx **trans_ctx,
- struct mail_index *index, const char *fname,
- int new_dir);
-int maildir_index_update_flags(struct mail_index *index,
- struct mail_index_record *rec, unsigned int seq,
- enum modify_type modify_type,
- enum mail_flags flags, int external_change);
-int maildir_try_flush_dirty_flags(struct mail_index *index, int force);
-
-struct istream *maildir_open_mail(struct mail_index *index,
- struct mail_index_record *rec,
- time_t *received_date, int *deleted);
-
-int maildir_expunge_mail(struct mail_index *index,
- struct mail_index_record *rec);
-
-void maildir_clean_tmp(const char *dir);
-
-#endif
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "istream.h"
-#include "maildir-index.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-
-static int do_open(struct mail_index *index, const char *path, void *context)
-{
- int *fd = context;
-
- *fd = open(path, O_RDONLY);
- if (*fd != -1)
- return 1;
- if (errno == ENOENT)
- return 0;
-
- index_file_set_syscall_error(index, path, "open()");
- return -1;
-}
-
-struct istream *maildir_open_mail(struct mail_index *index,
- struct mail_index_record *rec,
- time_t *received_date, int *deleted)
-{
- struct stat st;
- int fd;
-
- i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
-
- *deleted = FALSE;
-
- /* check for inconsistency here, to avoid extra error messages */
- if (index->inconsistent)
- return NULL;
-
- fd = -1;
- if (!maildir_file_do(index, rec, do_open, &fd))
- return NULL;
-
- if (fd == -1) {
- *deleted = TRUE;
- return NULL;
- }
-
- if (received_date != NULL) {
- if (fstat(fd, &st) == 0)
- *received_date = st.st_mtime;
- }
-
- if (index->mail_read_mmaped) {
- return i_stream_create_mmap(fd, default_pool,
- MAIL_MMAP_BLOCK_SIZE, 0, 0, TRUE);
- } else {
- return i_stream_create_file(fd, default_pool,
- MAIL_READ_BLOCK_SIZE, TRUE);
- }
-}
+++ /dev/null
-/* Copyright (C) 2002-2003 Timo Sirainen */
-
-/*
- Here's a description of how we handle Maildir synchronization and
- it's problems:
-
- We want to be as efficient as we can. The most efficient way to
- check if changes have occured is to stat() the new/ and cur/
- directories and uidlist file - if their mtimes haven't changed,
- there's no changes and we don't need to do anything.
-
- Problem 1: Multiple changes can happen within a single second -
- nothing guarantees that once we synced it, someone else didn't just
- then make a modification. Such modifications wouldn't get noticed
- until a new modification occured later.
-
- Problem 2: Syncing cur/ directory is much more costly than syncing
- new/. Moving mails from new/ to cur/ will always change mtime of
- cur/ causing us to sync it as well.
-
- Problem 3: We may not be able to move mail from new/ to cur/
- because we're out of quota, or simply because we're accessing a
- read-only mailbox.
-
-
- MAILDIR_SYNC_SECS
- -----------------
-
- Several checks below use MAILDIR_SYNC_SECS, which should be maximum
- clock drift between all computers accessing the maildir (eg. via
- NFS), rounded up to next second. Our default is 1 second, since
- everyone should be using NTP.
-
- Note that setting it to 0 works only if there's only one computer
- accessing the maildir. It's practically impossible to make two
- clocks _exactly_ synchronized.
-
- It might be possible to only use file server's clock by looking at
- the atime field, but I don't know how well that would actually work.
-
- cur directory
- -------------
-
- We have maildir_cur_dirty variable which is set to cur/ directory's
- mtime when it's >= time() - MAILDIR_SYNC_SECS and we _think_ we have
- synchronized the directory.
-
- When maildir_cur_dirty is non-zero, we don't synchronize the cur/
- directory until
-
- a) cur/'s mtime changes
- b) opening a mail fails with ENOENT
- c) time() > maildir_cur_dirty + MAILDIR_SYNC_SECS
-
- This allows us to modify the maildir multiple times without having
- to sync it at every change. The sync will eventually be done to
- make sure we didn't miss any external changes.
-
- The maildir_cur_dirty is set when:
-
- - we change message flags
- - we expunge messages
- - we move mail from new/ to cur/
- - we sync cur/ directory and it's mtime is
- >= time() - MAILDIR_SYNC_SECS
-
- It's unset when we do the final syncing, ie. when mtime is
- older than time() - MAILDIR_SYNC_SECS.
-
- new directory
- -------------
-
- If new/'s mtime is >= time() - MAILDIR_SYNC_SECS, always synchronize
- it. maildir_cur_dirty-like feature might save us a few syncs, but
- that might break a client which saves a mail in one connection and
- tries to fetch it in another one. new/ directory is almost always
- empty, so syncing it should be very fast anyway. Actually this can
- still happen if we sync only new/ dir while another client is also
- moving mails from it to cur/ - it takes us a while to see them.
- That's pretty unlikely to happen however, and only way to fix it
- would be to always synchronize cur/ after new/.
-
- Normally we move all mails from new/ to cur/ whenever we sync it. If
- it's not possible for some reason, we set maildir_have_new flag on
- which instructs synchronization to check files in new/ directory as
- well. maildir_keep_new flag is also set which instructs syncing to
- not even try to move mails to cur/ anymore.
-
- If client tries to change a flag for message in new/, we try to
- rename() it into cur/. If it's successful, we clear the
- maildir_keep_new flag so at next sync we'll try to move all of them
- to cur/. When all of them have been moved, maildir_have_new flag is
- cleared as well. Expunges will also clear maildir_keep_new flag.
-
- If rename() still fails because of ENOSPC or EDQUOT, we still save
- the flag changes in index with dirty-flag on. When moving the mail
- to cur/ directory, or when we notice it's already moved there, we
- apply the flag changes to the filename, rename it and remove the
- dirty flag. If there's dirty flags, this should be tried every time
- after expunge or when closing the mailbox.
-
- uidlist
- -------
-
- This file contains UID <-> filename mappings. It's updated only when
- new mail arrives, so it may contain filenames that have already been
- deleted. Updating is done by getting uidlist.lock file, writing the
- whole uidlist into it and rename()ing it over the old uidlist. This
- means there's no need to lock the file for reading.
-
- Whenever uidlist is rewritten, it's mtime must be larger than the old
- one's. Use utime() before rename() if needed.
-
- Only time you have to read this file is when assigning new UIDs for
- messages, to see if they already have UIDs. If file's mtime hasn't
- changed, you don't have to do even that.
-
- broken clients
- --------------
-
- Originally the middle identifier in Maildir filename was specified
- only as <process id>_<delivery counter>. That however created a
- problem with randomized PIDs which made it possible that the same
- PID was reused within one second.
-
- So if within one second a mail was delivered, MUA moved it to cur/
- and another mail was delivered by a new process using same PID as
- the first one, we likely ended up overwriting the first mail when
- the second mail was moved over it.
-
- Nowadays everyone should be giving a bit more specific identifier,
- for example include microseconds in it which Dovecot does.
-
- There's a simple way to prevent this from happening in some cases:
- Don't move the mail from new/ to cur/ if it's mtime is >= time() -
- MAILDIR_SYNC_SECS. The second delivery's link() call then fails
- because the file is already in new/, and it will then use a
- different filename. There's a few problems with this however:
-
- - while it's usually possible to read the mtime from beginning of
- the file name, it is against the Maildir specs. stat()ing the
- file then makes syncing slower.
- - another MUA might still move the mail to cur/
- - if first file's flags are modified by either Dovecot or another
- MUA, it's moved to cur/ (you _could_ just do the dirty-flagging
- but that'd be ugly)
-
- Because this is useful only for very few people and it requires some
- extra code, I decided not to implement it at least for now.
-
- It's also possible to never accidentally overwrite a mail by using
- link() + unlink() rather than rename(). This however isn't very
- good idea as it introduces potential race conditions when multiple
- clients are accessing the mailbox:
-
- Trying to move the same mail from new/ to cur/ at the same time:
-
- a) Client 1 uses slightly different filename than client 2,
- for example one sets read-flag on but the other doesn't.
- You have the same mail duplicated now.
-
- b) Client 3 sees the mail between Client 1's and 2's link() calls
- and changes it's flag. You have the same mail duplicated now.
-
- And it gets worse when they're unlink()ing in cur/ directory:
-
- c) Most other maildir clients use rename(). So if client 1 changes
- mail's flag with link()+unlink() and client 2 using rename()
- changes it back between 1's link() and unlink(), the mail gets
- expunged.
-
- d) If you try to deal with the duplicates by unlink()ing another
- one of them, you might end up unlinking both of them.
-
- So, what should we do then if we notice a duplicate? First of all,
- it might not be a duplicate at all, readdir() might have just
- returned it twice because it was just renamed. What we should do is
- create a completely new base name for it and rename() it to that.
- If the call fails with ENOENT, it only means that it wasn't a
- duplicate after all.
-*/
-
-#include "lib.h"
-#include "buffer.h"
-#include "istream.h"
-#include "hash.h"
-#include "ioloop.h"
-#include "str.h"
-#include "maildir-index.h"
-#include "maildir-uidlist.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <dirent.h>
-#include <utime.h>
-#include <sys/stat.h>
-
-#define MAILDIR_SYNC_SECS 1
-
-enum maildir_file_action {
- MAILDIR_FILE_ACTION_EXPUNGE,
- MAILDIR_FILE_ACTION_UPDATE_FLAGS,
- MAILDIR_FILE_ACTION_NEW,
- MAILDIR_FILE_ACTION_NONE,
-
- MAILDIR_FILE_FLAG_NEWDIR = 0x1000,
- MAILDIR_FILE_FLAG_ALLOCED = 0x2000,
- MAILDIR_FILE_FLAGS = 0x3000
-};
-
-struct maildir_hash_context {
- struct mail_index *index;
- struct mail_index_record *new_mail;
-
- int failed;
-};
-
-struct maildir_hash_rec {
- struct mail_index_record *rec;
- enum maildir_file_action action;
-};
-#define ACTION(hash) ((hash)->action & ~MAILDIR_FILE_FLAGS)
-
-struct maildir_sync_context {
- struct mail_index *index;
- const char *new_dir, *cur_dir;
-
- pool_t pool;
- struct hash_table *files;
- unsigned int new_count;
-
- DIR *new_dirp;
- struct dirent *new_dent;
-
- struct maildir_uidlist *uidlist;
- struct mail_cache_transaction_ctx *trans_ctx;
- unsigned int readonly_check:1;
- unsigned int flag_updates:1;
- unsigned int uidlist_rewrite:1;
- unsigned int new_mails_new:1;
- unsigned int new_mails_cur:1;
- unsigned int have_uncached_filenames:1;
-};
-
-static int maildir_sync_cur_dir(struct maildir_sync_context *ctx);
-
-/* a char* hash function from ASU -- from glib */
-static unsigned int maildir_hash(const void *p)
-{
- const unsigned char *s = p;
- unsigned int g, h = 0;
-
- while (*s != ':' && *s != '\0') {
- h = (h << 4) + *s;
- if ((g = h & 0xf0000000UL)) {
- h = h ^ (g >> 24);
- h = h ^ g;
- }
- s++;
- }
-
- return h;
-}
-
-static int maildir_cmp(const void *p1, const void *p2)
-{
- const char *s1 = p1, *s2 = p2;
-
- while (*s1 == *s2 && *s1 != ':' && *s1 != '\0') {
- s1++; s2++;
- }
- if ((*s1 == '\0' || *s1 == ':') &&
- (*s2 == '\0' || *s2 == ':'))
- return 0;
- return *s1 - *s2;
-}
-
-static int maildir_update_flags(struct maildir_sync_context *ctx,
- struct mail_index_record *rec,
- unsigned int seq, const char *new_fname)
-{
- enum mail_flags flags;
-
- if (ctx->index->lock_type != MAIL_LOCK_EXCLUSIVE)
- return TRUE;
-
- flags = maildir_filename_get_flags(new_fname, rec->msg_flags);
- flags &= ~ctx->index->private_flags_mask;
- flags |= rec->msg_flags & ctx->index->private_flags_mask;
-
- if (flags != rec->msg_flags) {
- if (!ctx->index->update_flags(ctx->index, rec,
- seq, MODIFY_REPLACE, flags, TRUE))
- return FALSE;
- }
-
- return TRUE;
-}
-
-static int maildir_sync_open_uidlist(struct maildir_sync_context *ctx)
-{
- struct mail_index *index = ctx->index;
- struct stat st;
- const char *path;
-
- if (ctx->uidlist != NULL)
- return TRUE;
-
- /* open it only if it's changed since we last synced it,
- or if we have uncached filenames. */
- path = t_strconcat(index->control_dir, "/" MAILDIR_UIDLIST_NAME, NULL);
- if (stat(path, &st) < 0) {
- if (errno == ENOENT) {
- /* doesn't exist yet, create it */
- switch (maildir_uidlist_try_lock(ctx->index)) {
- case -1:
- return FALSE;
- case 1:
- ctx->uidlist_rewrite = TRUE;
- break;
- }
-
- return TRUE;
- }
- return index_file_set_syscall_error(index, path, "stat()");
- }
-
- /* FIXME: last_uidlist_mtime should be in index headers */
- if (st.st_mtime == index->last_uidlist_mtime &&
- !ctx->have_uncached_filenames)
- return TRUE;
-
- ctx->uidlist = maildir_uidlist_open(index);
- if (ctx->uidlist == NULL)
- return TRUE;
-
- if (ctx->uidlist->uid_validity != index->header->uid_validity) {
- /* uidvalidity changed */
- if (!index->rebuilding && index->opened) {
- index_set_corrupted(index,
- "UIDVALIDITY changed in uidlist");
- return FALSE;
- }
-
- index->header->uid_validity = ctx->uidlist->uid_validity;
- i_assert(index->header->next_uid == 1);
- }
-
- if (index->header->next_uid > ctx->uidlist->next_uid) {
- index_set_corrupted(index, "index.next_uid (%u) > "
- "uidlist.next_uid (%u)",
- index->header->next_uid,
- ctx->uidlist->next_uid);
- return FALSE;
- }
-
- return TRUE;
-}
-
-static int maildir_time_cmp(const void *p1, const void *p2)
-{
- const char *s1 = *((const char **) p1);
- const char *s2 = *((const char **) p2);
- time_t t1 = 0, t2 = 0;
-
- /* we have to do numeric comparision, strcmp() will break when
- there's different amount of digits (mostly the 999999999 ->
- 1000000000 change in Sep 9 2001) */
- while (*s1 >= '0' && *s1 <= '9') {
- t1 = t1*10 + (*s1 - '0');
- s1++;
- }
- while (*s2 >= '0' && *s2 <= '9') {
- t2 = t2*10 + (*s2 - '0');
- s2++;
- }
-
- return t1 < t2 ? -1 : t1 > t2 ? 1 : 0;
-}
-
-static int maildir_full_sync_finish_new_mails(struct maildir_sync_context *ctx)
-{
- struct hash_iterate_context *iter;
- void *key, *value;
- const char *dir, **new_files;
- buffer_t *buf;
- unsigned int i;
- int new_dir;
-
- ctx->uidlist_rewrite = TRUE;
-
- /* then there's the completely new mails. sort them by the filename
- so we should get them to same order as they were created. */
- buf = buffer_create_static_hard(ctx->pool,
- ctx->new_count * sizeof(const char *));
- iter = hash_iterate_init(ctx->files);
- while (hash_iterate(iter, &key, &value)) {
- struct maildir_hash_rec *hash_rec = value;
-
- if (ACTION(hash_rec) == MAILDIR_FILE_ACTION_NEW) {
- buffer_append(buf, (const void *) &key,
- sizeof(const char *));
- }
- }
- hash_iterate_deinit(iter);
- i_assert(buffer_get_used_size(buf) ==
- ctx->new_count * sizeof(const char *));
-
- new_files = buffer_get_modifyable_data(buf, NULL);
- qsort(new_files, ctx->new_count, sizeof(const char *),
- maildir_time_cmp);
-
- if (!ctx->index->maildir_keep_new) {
- dir = ctx->cur_dir;
- new_dir = FALSE;
- } else {
- /* this is actually slightly wrong, because we don't really
- know if some of the new messages are in cur/ already.
- we could know that by saving it into buffer, but that'd
- require extra memory. luckily it doesn't really matter if
- we say it's in new/, but it's actually in cur/. we have
- to deal with such case anyway since another client might
- have just moved it. */
- dir = ctx->new_dir;
- new_dir = TRUE;
- ctx->index->maildir_have_new = TRUE;
- }
-
- for (i = 0; i < ctx->new_count; i++) {
- if (!maildir_index_append_file(&ctx->trans_ctx, ctx->index,
- new_files[i], new_dir))
- return FALSE;
- }
- ctx->new_count = 0;
-
- return TRUE;
-}
-
-static int maildir_full_sync_finish(struct maildir_sync_context *ctx)
-{
- struct mail_index *index = ctx->index;
- struct maildir_uidlist *uidlist;
- struct mail_index_record *rec, *first_rec, *last_rec;
- struct maildir_hash_rec *hash_rec;
- struct maildir_uidlist_rec uid_rec;
- enum maildir_file_action action;
- const char *fname, *dir;
- void *orig_key, *orig_value;
- unsigned int seq, first_seq, last_seq, uid, last_uid, new_flag;
- int new_dir, skip_next;
-
- if (ctx->new_count > 0) {
- /* new mails, either they're already in uidlist or we have
- to add them there. If we want to add them, we'll need to
- sync it locked. */
- if (maildir_uidlist_try_lock(ctx->index) < 0)
- return FALSE;
-
- if (!maildir_sync_open_uidlist(ctx))
- return FALSE;
- }
-
- seq = 1;
- rec = index->lookup(index, 1);
- uidlist = ctx->uidlist;
-
- if (uidlist == NULL)
- memset(&uid_rec, 0, sizeof(uid_rec));
- else {
- if (maildir_uidlist_next(uidlist, &uid_rec) < 0)
- return FALSE;
- }
-
- first_rec = last_rec = NULL;
- first_seq = last_seq = 0;
- skip_next = FALSE;
- while (rec != NULL) {
- uid = rec->uid;
-
- /* skip over the expunged records in uidlist */
- while (uid_rec.uid != 0 && uid_rec.uid < uid) {
- if (maildir_uidlist_next(uidlist, &uid_rec) < 0)
- return FALSE;
- }
-
- fname = maildir_get_location(index, rec, NULL);
- if (fname == NULL) {
- /* filename not cached, it must be in uidlist or
- it's expunged */
- fname = uid_rec.uid == rec->uid ?
- uid_rec.filename : NULL;
- }
-
- if (fname == NULL) {
- hash_rec = NULL;
- action = MAILDIR_FILE_ACTION_EXPUNGE;
- } else if (hash_lookup_full(ctx->files, fname,
- &orig_key, &orig_value)) {
- hash_rec = orig_value;
- action = ACTION(hash_rec);
- } else {
- /* none action */
- hash_rec = NULL;
- action = MAILDIR_FILE_ACTION_NONE;
- }
-
- if (uid_rec.uid == uid &&
- maildir_cmp(fname, uid_rec.filename) != 0) {
- index_set_corrupted(index,
- "Filename mismatch for UID %u: %s vs %s",
- uid, fname, uid_rec.filename);
- return FALSE;
- }
-
- if (uid_rec.uid > uid && hash_rec != NULL &&
- (action == MAILDIR_FILE_ACTION_UPDATE_FLAGS ||
- action == MAILDIR_FILE_ACTION_NONE)) {
- /* it's UID has changed. shouldn't happen. */
- index_set_corrupted(index,
- "UID changed for %s/%s: %u -> %u",
- index->mailbox_path, fname,
- uid, uid_rec.uid);
- return FALSE;
- }
-
- switch (action) {
- case MAILDIR_FILE_ACTION_EXPUNGE:
- if (first_rec == NULL) {
- first_rec = rec;
- first_seq = seq;
- }
- last_rec = rec;
- last_seq = seq;
- break;
- case MAILDIR_FILE_ACTION_NEW:
- /* filename wasn't cached */
- new_flag = hash_rec->action & MAILDIR_FILE_FLAG_NEWDIR;
- hash_rec->action = MAILDIR_FILE_ACTION_NONE | new_flag;
- ctx->new_count--;
-
- if (!maildir_cache_update_file(&ctx->trans_ctx, index,
- rec, fname, new_flag))
- return FALSE;
- /* fall through */
- case MAILDIR_FILE_ACTION_UPDATE_FLAGS:
- new_dir = (hash_rec->action &
- MAILDIR_FILE_FLAG_NEWDIR) != 0;
- maildir_index_update_filename(index, rec->uid,
- orig_key, new_dir);
- if (!maildir_update_flags(ctx, rec, seq, orig_key))
- return FALSE;
- /* fall through */
- case MAILDIR_FILE_ACTION_NONE:
- if (first_rec != NULL) {
- if (!index->expunge(index, first_rec, last_rec,
- first_seq, last_seq, TRUE))
- return FALSE;
- first_rec = NULL;
-
- seq = first_seq;
- rec = index->lookup(index, seq);
- skip_next = TRUE;
- }
- break;
- default:
- i_unreached();
- }
-
- if (uid_rec.uid == uid) {
- if (maildir_uidlist_next(uidlist, &uid_rec) < 0)
- return FALSE;
- }
-
- if (skip_next)
- skip_next = FALSE;
- else {
- rec = index->next(index, rec);
- seq++;
- }
- }
-
- if (first_rec != NULL) {
- if (!index->expunge(index, first_rec, last_rec,
- first_seq, last_seq, TRUE))
- return FALSE;
- seq = first_seq;
- }
-
- if (seq-1 != index->header->messages_count) {
- index_set_corrupted(index,
- "Wrong messages_count in header (%u != %u)",
- seq, index->header->messages_count);
- return FALSE;
- }
-
- /* if there's new mails which are already in uidlist, get them */
- last_uid = 0;
- while (uid_rec.uid != 0) {
- if (hash_lookup_full(ctx->files, uid_rec.filename,
- &orig_key, &orig_value))
- hash_rec = orig_value;
- else
- hash_rec = NULL;
-
- if (hash_rec != NULL &&
- ACTION(hash_rec) == MAILDIR_FILE_ACTION_NONE) {
- /* it's a duplicate, shouldn't happen */
- i_error("%s: Found duplicate filename %s, rebuilding",
- ctx->uidlist->fname, uid_rec.filename);
- (void)unlink(ctx->uidlist->fname);
-
- if (INDEX_IS_UIDLIST_LOCKED(index))
- ctx->uidlist_rewrite = TRUE;
- hash_rec = NULL;
- }
-
- if (hash_rec != NULL) {
- i_assert(ACTION(hash_rec) == MAILDIR_FILE_ACTION_NEW);
-
- /* make sure we set the same UID for it. */
- if (index->header->next_uid > uid_rec.uid) {
- index_set_corrupted(index,
- "index.next_uid (%u) > "
- "uid_rec.uid (%u)",
- index->header->next_uid,
- uid_rec.uid);
- return FALSE;
- }
- index->header->next_uid = uid_rec.uid;
-
- new_flag = hash_rec->action & MAILDIR_FILE_FLAG_NEWDIR;
- hash_rec->action = MAILDIR_FILE_ACTION_NONE | new_flag;
- ctx->new_count--;
-
- if (new_flag != 0)
- ctx->index->maildir_have_new = TRUE;
- dir = new_flag != 0 ? ctx->new_dir : ctx->cur_dir;
-
- if (!maildir_index_append_file(&ctx->trans_ctx, index,
- orig_key, new_flag != 0))
- return FALSE;
- }
-
- if (maildir_uidlist_next(uidlist, &uid_rec) < 0)
- return FALSE;
- }
-
- if (ctx->uidlist != NULL) {
- /* update our next_uid. it should have been checked for
- sanity already. */
- struct stat st;
-
- i_assert(index->header->next_uid <= ctx->uidlist->next_uid);
- index->header->next_uid = ctx->uidlist->next_uid;
-
- /* uidlist is now synced, remember that. */
- if (fstat(i_stream_get_fd(ctx->uidlist->input), &st) < 0) {
- return index_file_set_syscall_error(index,
- ctx->uidlist->fname,
- "fstat()");
- }
- index->last_uidlist_mtime = st.st_mtime;
- }
-
- if (ctx->new_count > 0 && INDEX_IS_UIDLIST_LOCKED(index))
- maildir_full_sync_finish_new_mails(ctx);
-
- /* all done (or can't do it since we don't have lock) */
- ctx->index->maildir_synced_once = TRUE;
- if (ctx->trans_ctx != NULL)
- mail_cache_transaction_commit(ctx->trans_ctx);
- return TRUE;
-}
-
-static int maildir_full_sync_init(struct maildir_sync_context *ctx,
- int only_new)
-{
- struct mail_index *index = ctx->index;
- struct mail_index_record *rec;
- struct maildir_hash_rec *hash_rec;
- const char *fname;
- size_t size;
- int new_dir, have_new;
-
- if (index->header->messages_count >= INT_MAX/32) {
- index_set_corrupted(index, "Header says %u messages",
- index->header->messages_count);
- return FALSE;
- }
-
- /* we're resyncing everything, so reset the filename hash */
- if (index->new_filenames != NULL) {
- hash_destroy(index->new_filenames);
- index->new_filenames = NULL;
- }
-
- if (index->new_filename_pool != NULL)
- p_clear(index->new_filename_pool);
-
- /* reset synced-flag too, just in case something fails and we don't
- have up-to-date new_filenames */
- ctx->index->maildir_synced_once = FALSE;
-
- /* read current messages in index into hash */
- size = nearest_power(index->header->messages_count *
- sizeof(struct maildir_hash_rec) + 1024);
- ctx->pool = pool_alloconly_create("maildir sync", I_MAX(size, 16384));
- ctx->files = hash_create(default_pool, ctx->pool,
- index->header->messages_count * 2,
- maildir_hash, maildir_cmp);
- ctx->new_count = 0;
-
- have_new = FALSE;
-
- /* Now we'll fill the hash with cached filenames. This is done mostly
- just to save some memory since we can use pointers to mmaped cache
- file. Note that all records may not have the filename cached.
-
- WARNING: Cache file must not be modified as long as these pointers
- exist, as modifying might change the mmap base address. The call
- below makes sure that cache file is initially fully mmaped. */
- if (mail_cache_get_mmaped(index->cache, &size) == NULL)
- return FALSE;
-
- rec = index->lookup(index, 1);
- while (rec != NULL) {
- fname = maildir_get_location(index, rec, &new_dir);
- if (fname == NULL)
- ctx->have_uncached_filenames = TRUE;
-
- if (new_dir)
- have_new = TRUE;
-
- if ((!only_new || new_dir) && fname != NULL) {
- hash_rec = p_new(ctx->pool, struct maildir_hash_rec, 1);
- hash_rec->rec = rec;
- hash_rec->action = MAILDIR_FILE_ACTION_EXPUNGE;
-
- if (hash_lookup(ctx->files, fname) != NULL) {
- index_set_corrupted(index,
- "Duplicated message %s", fname);
- return FALSE;
- }
-
- hash_insert(ctx->files, (void *) fname, hash_rec);
- }
-
- rec = index->next(index, rec);
- }
-
- index->maildir_have_new = have_new;
- return TRUE;
-}
-
-static int maildir_fix_duplicate(struct mail_index *index,
- const char *old_fname, int new_dir)
-{
- const char *new_fname, *old_path, *new_path;
- int ret = TRUE;
-
- t_push();
-
- old_path = t_strconcat(index->mailbox_path, new_dir ? "/new/" : "/cur/",
- old_fname, NULL);
-
- new_fname = maildir_generate_tmp_filename(&ioloop_timeval);
- new_path = t_strconcat(index->mailbox_path, "/new/", new_fname, NULL);
-
- if (rename(old_path, new_path) == 0) {
- i_warning("Fixed duplicate in %s: %s -> %s",
- index->mailbox_path, old_fname, new_fname);
- } else if (errno != ENOENT) {
- index_set_error(index, "rename(%s, %s) failed: %m",
- old_path, new_path);
- ret = FALSE;
- }
- t_pop();
-
- return ret;
-}
-
-static int maildir_full_sync_dir(struct maildir_sync_context *ctx,
- int new_dir, DIR *dirp, struct dirent *d)
-{
- struct hash_iterate_context *iter;
- void *key, *value;
- struct maildir_hash_rec *hash_rec;
- void *orig_key, *orig_value;
- int newflag;
-
- newflag = new_dir ? MAILDIR_FILE_FLAG_NEWDIR : 0;
-
- do {
- if (d->d_name[0] == '.')
- continue;
-
- if (!hash_lookup_full(ctx->files, d->d_name,
- &orig_key, &orig_value)) {
- hash_rec = p_new(ctx->pool, struct maildir_hash_rec, 1);
- } else {
- hash_rec = orig_value;
- if (ACTION(hash_rec) != MAILDIR_FILE_ACTION_EXPUNGE) {
- if (!maildir_fix_duplicate(ctx->index,
- d->d_name, new_dir))
- return FALSE;
- continue;
- }
- }
-
- if (hash_rec->rec == NULL) {
- /* new message */
- if (ctx->readonly_check &&
- !ctx->have_uncached_filenames)
- continue;
-
- if (new_dir)
- ctx->new_mails_new = TRUE;
- else
- ctx->new_mails_cur = TRUE;
-
- ctx->new_count++;
- hash_rec->action = MAILDIR_FILE_ACTION_NEW | newflag;
- hash_insert(ctx->files, p_strdup(ctx->pool, d->d_name),
- hash_rec);
- continue;
- }
-
- if (strcmp(orig_key, d->d_name) != 0) {
- hash_rec->action =
- MAILDIR_FILE_ACTION_UPDATE_FLAGS | newflag;
-
- hash_insert(ctx->files, p_strdup(ctx->pool, d->d_name),
- hash_rec);
- ctx->flag_updates = TRUE;
- } else {
- hash_rec->action = MAILDIR_FILE_ACTION_NONE | newflag;
- }
- } while ((d = readdir(dirp)) != NULL);
-
- /* records that are left to hash must not have any (filename) pointers
- to cache file. So remove none actions, and p_strdup() expunge
- actions. */
- iter = hash_iterate_init(ctx->files);
- while (hash_iterate(iter, &key, &value)) {
- struct maildir_hash_rec *hash_rec = value;
-
- switch (ACTION(hash_rec)) {
- case MAILDIR_FILE_ACTION_NONE:
- hash_remove(ctx->files, key);
- break;
- case MAILDIR_FILE_ACTION_EXPUNGE:
- if (hash_rec->action & MAILDIR_FILE_FLAG_ALLOCED) {
- /* we're getting here because our recently
- inserted node is traversed as well */
- break;
- }
-
- hash_rec->action |= MAILDIR_FILE_FLAG_ALLOCED;
- hash_insert(ctx->files,
- p_strdup(ctx->pool, key), value);
- break;
- default:
- break;
- }
- }
- hash_iterate_deinit(iter);
-
- return TRUE;
-}
-
-static int maildir_new_scan_first_file(struct maildir_sync_context *ctx)
-{
- DIR *dirp;
- struct dirent *d;
-
- dirp = opendir(ctx->new_dir);
- if (dirp == NULL) {
- return index_file_set_syscall_error(ctx->index, ctx->new_dir,
- "opendir()");
- }
-
- /* find first file */
- while ((d = readdir(dirp)) != NULL) {
- if (d->d_name[0] != '.')
- break;
- }
-
- if (d == NULL) {
- if (closedir(dirp) < 0) {
- index_file_set_syscall_error(ctx->index, ctx->new_dir,
- "closedir()");
- }
- } else {
- ctx->new_dirp = dirp;
- ctx->new_dent = d;
- }
-
- return TRUE;
-}
-
-static int maildir_full_sync_dirs(struct maildir_sync_context *ctx)
-{
- DIR *dirp;
- int failed;
-
- if (ctx->new_dirp == NULL &&
- (ctx->index->maildir_have_new || ctx->index->maildir_keep_new)) {
- if (!maildir_new_scan_first_file(ctx))
- return FALSE;
- }
-
- if (ctx->new_dent != NULL) {
- if (!maildir_full_sync_dir(ctx, TRUE, ctx->new_dirp,
- ctx->new_dent))
- return FALSE;
- ctx->new_dent = NULL;
- }
-
- dirp = opendir(ctx->cur_dir);
- if (dirp == NULL) {
- return index_file_set_syscall_error(ctx->index, ctx->cur_dir,
- "opendir()");
- }
-
- failed = !maildir_full_sync_dir(ctx, FALSE, dirp, readdir(dirp));
-
- if (closedir(dirp) < 0) {
- return index_file_set_syscall_error(ctx->index, ctx->cur_dir,
- "closedir()");
- }
-
- return !failed;
-}
-
-static int maildir_sync_new_dir_full(struct maildir_sync_context *ctx)
-{
- if (!ctx->index->set_lock(ctx->index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
-
- if (!maildir_full_sync_init(ctx, TRUE))
- return FALSE;
-
- if (!maildir_full_sync_dir(ctx, TRUE, ctx->new_dirp, ctx->new_dent))
- return FALSE;
- ctx->new_dent = NULL;
-
- if (!maildir_full_sync_finish(ctx))
- return FALSE;
-
- return TRUE;
-}
-
-static int maildir_sync_new_dir(struct maildir_sync_context *ctx,
- int move_to_cur, int append_index)
-{
- struct dirent *d;
- string_t *sourcepath, *destpath;
- const char *final_dir;
-
- if (append_index) {
- if (ctx->index->maildir_have_new) {
- /* some of the mails in new/ are already indexed.
- we'll have to do a full sync. */
- return maildir_sync_new_dir_full(ctx);
- }
-
- if (!ctx->index->set_lock(ctx->index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
-
- switch (maildir_uidlist_try_lock(ctx->index)) {
- case -1:
- return FALSE;
- case 0:
- /* couldn't get a lock.
- no point in doing more. */
- return TRUE;
- }
-
- /* make sure uidlist is up to date.
- if it's not, do a full sync. */
- if (!maildir_sync_open_uidlist(ctx))
- return FALSE;
-
- if (ctx->uidlist != NULL)
- return maildir_sync_cur_dir(ctx);
-
- ctx->uidlist_rewrite = TRUE;
- }
-
- d = ctx->new_dent;
- ctx->new_dent = NULL;
-
- sourcepath = t_str_new(PATH_MAX);
- destpath = t_str_new(PATH_MAX);
-
- final_dir = move_to_cur ? ctx->cur_dir : ctx->new_dir;
-
- do {
- if (d->d_name[0] == '.')
- continue;
-
- str_truncate(sourcepath, 0);
- str_printfa(sourcepath, "%s/%s", ctx->new_dir, d->d_name);
-
- if (move_to_cur) {
- str_truncate(destpath, 0);
- str_printfa(destpath, "%s/%s", ctx->cur_dir, d->d_name);
-
- if (rename(str_c(sourcepath), str_c(destpath)) < 0 &&
- errno != ENOENT) {
- if (ENOSPACE(errno))
- ctx->index->nodiskspace = TRUE;
- else if (errno == EACCES)
- ctx->index->mailbox_readonly = TRUE;
- else {
- index_set_error(ctx->index,
- "rename(%s, %s) failed: %m",
- str_c(sourcepath),
- str_c(destpath));
- return FALSE;
- }
-
- ctx->index->maildir_keep_new = TRUE;
- if (!append_index) {
- ctx->new_dent = d;
- return TRUE;
- }
-
- /* continue by keeping them in new/ dir */
- final_dir = ctx->new_dir;
- move_to_cur = FALSE;
- }
- }
-
- if (append_index) {
- if (!move_to_cur)
- ctx->index->maildir_have_new = TRUE;
-
- t_push();
- if (!maildir_index_append_file(&ctx->trans_ctx,
- ctx->index, d->d_name,
- !move_to_cur)) {
- t_pop();
- return FALSE;
- }
- t_pop();
- }
- } while ((d = readdir(ctx->new_dirp)) != NULL);
-
- return TRUE;
-}
-
-static int maildir_sync_cur_dir(struct maildir_sync_context *ctx)
-{
- struct mail_index *index = ctx->index;
-
- if (ctx->new_dent != NULL && !index->maildir_keep_new) {
- /* there's also new mails. move them into cur/ first, if we
- can lock the uidlist */
- switch (maildir_uidlist_try_lock(index)) {
- case -1:
- return FALSE;
- case 1:
- if (!maildir_sync_new_dir(ctx, TRUE, FALSE))
- return FALSE;
- }
- }
-
- if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
-
- if (!maildir_full_sync_init(ctx, FALSE) ||
- !maildir_full_sync_dirs(ctx) ||
- !maildir_full_sync_finish(ctx))
- return FALSE;
-
- return TRUE;
-}
-
-static int maildir_index_sync_context(struct maildir_sync_context *ctx,
- int *changes)
-
-{
- struct mail_index *index = ctx->index;
- struct stat st;
- time_t new_mtime, cur_mtime;
-
- if (!maildir_try_flush_dirty_flags(ctx->index, FALSE))
- return FALSE;
-
- if (stat(ctx->new_dir, &st) < 0) {
- index_file_set_syscall_error(index, ctx->new_dir, "stat()");
- return FALSE;
- }
- new_mtime = st.st_mtime;
-
- if (stat(ctx->cur_dir, &st) < 0) {
- index_file_set_syscall_error(index, ctx->cur_dir, "stat()");
- return FALSE;
- }
- cur_mtime = st.st_mtime;
-
- if (new_mtime != index->last_new_mtime ||
- new_mtime >= ioloop_time - MAILDIR_SYNC_SECS) {
- if (!maildir_new_scan_first_file(ctx))
- return FALSE;
- }
-
- if (cur_mtime != index->sync_stamp &&
- index->sync_dirty_stamp == 0) {
- /* update index->sync_stamp from header.
- set_lock() does it automatically. */
- if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
- }
-
- if (cur_mtime != index->sync_stamp ||
- (index->sync_dirty_stamp != 0 &&
- index->sync_dirty_stamp < ioloop_time - MAILDIR_SYNC_SECS)) {
- /* cur/ changed, or delayed cur/ check */
- if (changes != NULL)
- *changes = TRUE;
-
- if (!maildir_sync_cur_dir(ctx))
- return FALSE;
- }
-
- if (ctx->new_dent != NULL) {
- if (changes != NULL)
- *changes = TRUE;
-
- if (!maildir_sync_new_dir(ctx, !index->maildir_keep_new, TRUE))
- return FALSE;
-
- /* this will set maildir_cur_dirty. it may actually be
- different from cur/'s mtime if we're unlucky, but that only
- causes extra sync and it's not worth the extra stat() */
- if (ctx->new_dent == NULL &&
- (ctx->new_count == 0 || !ctx->new_mails_new))
- cur_mtime = time(NULL);
- }
-
- if (ctx->uidlist_rewrite) {
- i_assert(INDEX_IS_UIDLIST_LOCKED(index));
-
- if (!maildir_uidlist_rewrite(index, &index->last_uidlist_mtime))
- return FALSE;
- }
-
- if (index->lock_type == MAIL_LOCK_EXCLUSIVE) {
- if (index->maildir_have_new)
- index->header->flags |= MAIL_INDEX_FLAG_MAILDIR_NEW;
- else
- index->header->flags &= ~MAIL_INDEX_FLAG_MAILDIR_NEW;
- }
-
- if (index->sync_dirty_stamp == 0 ||
- index->sync_dirty_stamp < ioloop_time - MAILDIR_SYNC_SECS) {
- if (cur_mtime >= ioloop_time - MAILDIR_SYNC_SECS)
- index->sync_dirty_stamp = cur_mtime;
- else if (ctx->new_count == 0 || !ctx->new_mails_cur)
- index->sync_dirty_stamp = 0;
- else {
- /* uidlist is locked, wait for a while before
- trying again */
- index->sync_dirty_stamp = ioloop_time;
- }
- }
-
- index->sync_stamp = cur_mtime;
- if (ctx->new_dent == NULL &&
- (ctx->new_count == 0 || !ctx->new_mails_new))
- index->last_new_mtime = new_mtime;
-
- return TRUE;
-}
-
-static int maildir_full_sync_finish_readonly(struct maildir_sync_context *ctx)
-{
- struct mail_index *index = ctx->index;
- struct mail_index_record *rec;
- struct maildir_hash_rec *hash_rec;
- struct maildir_uidlist *uidlist;
- struct maildir_uidlist_rec uid_rec;
- void *orig_key, *orig_value;
- const char *fname;
- unsigned int seq;
- int new_dir, tried_uidlist;
-
- if (!ctx->flag_updates && !ctx->have_uncached_filenames) {
- ctx->index->maildir_synced_once = TRUE;
- return TRUE;
- }
-
- memset(&uid_rec, 0, sizeof(uid_rec));
- uidlist = ctx->uidlist;
- tried_uidlist = FALSE;
-
- rec = index->lookup(index, 1); seq = 1;
- for (; rec != NULL; rec = index->next(index, rec), seq++) {
- fname = maildir_get_location(index, rec, NULL);
- if (fname == NULL) {
- /* not cached, get it from uidlist */
- if (uidlist == NULL && !tried_uidlist) {
- ctx->have_uncached_filenames = TRUE;
- if (!maildir_sync_open_uidlist(ctx))
- return FALSE;
-
- uidlist = ctx->uidlist;
- tried_uidlist = TRUE;
-
- /* get the initial record */
- if (uidlist != NULL &&
- maildir_uidlist_next(uidlist, &uid_rec) < 0)
- return FALSE;
- }
-
- if (uidlist == NULL) {
- /* uidlist doesn't exist? shouldn't happen */
- continue;
- }
-
- while (uid_rec.uid != 0 && uid_rec.uid < rec->uid) {
- if (maildir_uidlist_next(uidlist, &uid_rec) < 0)
- return FALSE;
- }
-
- if (uid_rec.uid != rec->uid) {
- /* not in uidlist, it's expunged */
- continue;
- }
-
- fname = uid_rec.filename;
- }
-
- if (!hash_lookup_full(ctx->files, fname,
- &orig_key, &orig_value))
- continue;
-
- hash_rec = orig_value;
- if (ACTION(hash_rec) != MAILDIR_FILE_ACTION_UPDATE_FLAGS &&
- ACTION(hash_rec) != MAILDIR_FILE_ACTION_NEW)
- continue;
-
- new_dir = (hash_rec->action & MAILDIR_FILE_FLAG_NEWDIR) != 0;
- maildir_index_update_filename(index, rec->uid,
- orig_key, new_dir);
-
- if (!maildir_update_flags(ctx, rec, seq, orig_key))
- return FALSE;
- }
-
- ctx->index->maildir_synced_once = TRUE;
- return TRUE;
-}
-
-static int maildir_index_sync_context_readonly(struct maildir_sync_context *ctx)
-{
- struct mail_index *index = ctx->index;
- struct stat st;
- int cur_changed;
-
- i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
-
- if (!index->maildir_synced_once) {
- /* we haven't synced yet in this session. do it */
- cur_changed = TRUE;
- } else {
- if (stat(ctx->cur_dir, &st) < 0) {
- index_file_set_syscall_error(index, ctx->cur_dir,
- "stat()");
- return FALSE;
- }
-
- cur_changed = st.st_mtime != index->sync_stamp ||
- index->sync_dirty_stamp != 0;
- }
-
- if (!cur_changed) {
- if (!index->maildir_have_new) {
- /* no changes */
- return TRUE;
- }
-
- if (stat(ctx->new_dir, &st) < 0) {
- return index_file_set_syscall_error(index, ctx->new_dir,
- "stat()");
- }
- if (st.st_mtime == index->last_new_mtime &&
- st.st_mtime < ioloop_time - MAILDIR_SYNC_SECS) {
- /* no changes */
- return TRUE;
- }
-
- if (!maildir_new_scan_first_file(ctx))
- return FALSE;
- }
-
- /* ok, something's changed. check only changes in file names. */
-
- /* if we can get exclusive lock, we can update the index
- directly. but don't rely on it. */
- (void)index->try_lock(index, MAIL_LOCK_EXCLUSIVE);
-
- if (!maildir_full_sync_init(ctx, FALSE) ||
- !maildir_full_sync_dirs(ctx) ||
- !maildir_full_sync_finish_readonly(ctx))
- return FALSE;
-
- return TRUE;
-}
-
-static void maildir_index_sync_deinit(struct maildir_sync_context *ctx)
-{
- // FIXME: remove new flags from cache if needed
- if (ctx->trans_ctx != NULL)
- mail_cache_transaction_end(ctx->trans_ctx);
- if (ctx->uidlist != NULL)
- maildir_uidlist_close(ctx->uidlist);
- if (ctx->files != NULL)
- hash_destroy(ctx->files);
- if (ctx->pool != NULL)
- pool_unref(ctx->pool);
-
- if (ctx->new_dirp != NULL) {
- if (closedir(ctx->new_dirp) < 0) {
- index_file_set_syscall_error(ctx->index, ctx->new_dir,
- "closedir()");
- }
- }
-
- maildir_uidlist_unlock(ctx->index);
-}
-
-static struct maildir_sync_context *
-maildir_sync_context_new(struct mail_index *index)
-{
- struct maildir_sync_context *ctx;
-
- ctx = t_new(struct maildir_sync_context, 1);
- ctx->index = index;
- ctx->new_dir = t_strconcat(index->mailbox_path, "/new", NULL);
- ctx->cur_dir = t_strconcat(index->mailbox_path, "/cur", NULL);
- return ctx;
-}
-
-int maildir_index_sync_readonly(struct mail_index *index,
- const char *fname, int *found)
-{
- struct maildir_sync_context *ctx;
- struct maildir_hash_rec *hash_rec;
- int ret;
-
- ctx = maildir_sync_context_new(index);
- ctx->readonly_check = TRUE;
-
- ret = maildir_index_sync_context_readonly(ctx);
-
- if (!ret || ctx->files == NULL || fname == NULL)
- *found = FALSE;
- else {
- hash_rec = hash_lookup(ctx->files, fname);
- *found = hash_rec != NULL &&
- hash_rec->action != MAILDIR_FILE_ACTION_EXPUNGE;
- }
- maildir_index_sync_deinit(ctx);
- return ret;
-}
-
-int maildir_index_sync(struct mail_index *index, int minimal_sync,
- enum mail_lock_type data_lock_type __attr_unused__,
- int *changes)
-{
- struct maildir_sync_context *ctx;
- int ret;
-
- i_assert(index->lock_type != MAIL_LOCK_SHARED);
-
- if (changes != NULL)
- *changes = FALSE;
-
- if (minimal_sync)
- return TRUE;
-
- ctx = maildir_sync_context_new(index);
- ret = maildir_index_sync_context(ctx, changes);
- maildir_index_sync_deinit(ctx);
- return ret;
-}
+++ /dev/null
-/* Copyright (C) 2003 Timo Sirainen */
-
-#include "lib.h"
-#include "ioloop.h"
-#include "istream.h"
-#include "str.h"
-#include "write-full.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-#include "maildir-index.h"
-#include "maildir-uidlist.h"
-
-#include <stdio.h>
-#include <sys/stat.h>
-#include <utime.h>
-
-/* how many seconds to wait before overriding uidlist.lock */
-#define UIDLIST_LOCK_STALE_TIMEOUT (60*5)
-
-int maildir_uidlist_try_lock(struct mail_index *index)
-{
- const char *path;
- mode_t old_mask;
- int fd;
-
- if (INDEX_IS_UIDLIST_LOCKED(index))
- return 1;
-
- path = t_strconcat(index->control_dir, "/" MAILDIR_UIDLIST_NAME, NULL);
- old_mask = umask(0777 & ~index->mail_create_mode);
- fd = file_dotlock_open(path, NULL, 0, 0, UIDLIST_LOCK_STALE_TIMEOUT,
- NULL, NULL);
- umask(old_mask);
- if (fd == -1) {
- if (errno == EAGAIN)
- return 0;
- return -1;
- }
-
- index->maildir_lock_fd = fd;
- return 1;
-}
-
-void maildir_uidlist_unlock(struct mail_index *index)
-{
- const char *path;
-
- if (!INDEX_IS_UIDLIST_LOCKED(index))
- return;
-
- path = t_strconcat(index->control_dir, "/" MAILDIR_UIDLIST_NAME, NULL);
- (void)file_dotlock_delete(path, index->maildir_lock_fd);
- index->maildir_lock_fd = -1;
-}
-
-struct maildir_uidlist *maildir_uidlist_open(struct mail_index *index)
-{
- const char *path, *line;
- struct maildir_uidlist *uidlist;
- unsigned int version;
- int fd;
-
- path = t_strconcat(index->control_dir, "/" MAILDIR_UIDLIST_NAME, NULL);
- fd = open(path, O_RDONLY);
- if (fd == -1) {
- if (errno != ENOENT)
- index_file_set_syscall_error(index, path, "open()");
- return NULL;
- }
-
- uidlist = i_new(struct maildir_uidlist, 1);
- uidlist->index = index;
- uidlist->fname = i_strdup(path);
- uidlist->input = i_stream_create_file(fd, default_pool, 4096, TRUE);
-
- /* get header */
- line = i_stream_read_next_line(uidlist->input);
- if (line == NULL || sscanf(line, "%u %u %u", &version,
- &uidlist->uid_validity,
- &uidlist->next_uid) != 3 ||
- version != 1) {
- /* broken file */
- (void)unlink(path);
- maildir_uidlist_close(uidlist);
- return NULL;
- }
-
- return uidlist;
-}
-
-int maildir_uidlist_next(struct maildir_uidlist *uidlist,
- struct maildir_uidlist_rec *uid_rec)
-{
- const char *line;
- unsigned int uid;
-
- memset(uid_rec, 0, sizeof(*uid_rec));
-
- line = i_stream_read_next_line(uidlist->input);
- if (line == NULL)
- return 0;
-
- uid = 0;
- while (*line >= '0' && *line <= '9') {
- uid = uid*10 + (*line - '0');
- line++;
- }
-
- if (uid == 0 || *line != ' ') {
- /* invalid file */
- index_set_error(uidlist->index, "Invalid data in file %s",
- uidlist->fname);
- (void)unlink(uidlist->fname);
- return -1;
- }
- if (uid <= uidlist->last_read_uid) {
- index_set_error(uidlist->index,
- "UIDs not ordered in file %s (%u > %u)",
- uidlist->fname, uid, uidlist->last_read_uid);
- (void)unlink(uidlist->fname);
- return -1;
- }
- if (uid >= uidlist->next_uid) {
- index_set_error(uidlist->index,
- "UID larger than next_uid in file %s "
- "(%u >= %u)", uidlist->fname,
- uid, uidlist->next_uid);
- (void)unlink(uidlist->fname);
- return -1;
- }
-
- while (*line == ' ') line++;
-
- uid_rec->uid = uid;
- uid_rec->filename = line;
- return 1;
-}
-
-void maildir_uidlist_close(struct maildir_uidlist *uidlist)
-{
- i_stream_unref(uidlist->input);
- i_free(uidlist->fname);
- i_free(uidlist);
-}
-
-static int maildir_uidlist_rewrite_fd(struct mail_index *index,
- const char *temp_path, time_t *mtime)
-{
- struct mail_index_record *rec;
- struct utimbuf ut;
- const char *p, *fname;
- string_t *str;
- size_t len;
-
- str = t_str_new(4096);
- str_printfa(str, "1 %u %u\n",
- index->header->uid_validity, index->header->next_uid);
-
- rec = index->lookup(index, 1);
- while (rec != NULL) {
- fname = maildir_get_location(index, rec, NULL);
- /* maildir should be synced, so above call should never fail */
- i_assert(fname != NULL);
-
- p = strchr(fname, ':');
- len = p == NULL ? strlen(fname) : (size_t)(p-fname);
-
- if (str_len(str) + MAX_INT_STRLEN + len + 2 >= 4096) {
- /* flush buffer */
- if (write_full(index->maildir_lock_fd,
- str_data(str), str_len(str)) < 0) {
- index_file_set_syscall_error(index, temp_path,
- "write_full()");
- return FALSE;
- }
- str_truncate(str, 0);
- }
-
- str_printfa(str, "%u ", rec->uid);
- str_append_n(str, fname, len);
- str_append_c(str, '\n');
-
- rec = index->next(index, rec);
- }
-
- if (write_full(index->maildir_lock_fd,
- str_data(str), str_len(str)) < 0) {
- index_file_set_syscall_error(index, temp_path, "write_full()");
- return FALSE;
- }
-
- /* uidlist's mtime must grow every time */
- *mtime = ioloop_time > *mtime ? ioloop_time : *mtime + 1;
- ut.actime = ioloop_time;
- ut.modtime = *mtime;
- if (utime(temp_path, &ut) < 0)
- index_set_syscall_error(index, "utime()");
-
- if (fsync(index->maildir_lock_fd) < 0) {
- index_file_set_syscall_error(index, temp_path, "fsync()");
- return FALSE;
- }
-
- return TRUE;
-}
-
-int maildir_uidlist_rewrite(struct mail_index *index, time_t *mtime)
-{
- const char *temp_path, *db_path;
- int failed = FALSE;
-
- i_assert(INDEX_IS_UIDLIST_LOCKED(index));
-
- if (index->lock_type == MAIL_LOCK_UNLOCK) {
- if (!index->set_lock(index, MAIL_LOCK_SHARED))
- return FALSE;
- }
-
- temp_path = t_strconcat(index->control_dir,
- "/" MAILDIR_UIDLIST_NAME ".lock", NULL);
-
- failed = !maildir_uidlist_rewrite_fd(index, temp_path, mtime);
-
- if (!failed) {
- db_path = t_strconcat(index->control_dir,
- "/" MAILDIR_UIDLIST_NAME, NULL);
-
- if (file_dotlock_replace(db_path, index->maildir_lock_fd,
- FALSE) <= 0) {
- index_set_error(index,
- "file_dotlock_replace(%s) failed: %m",
- db_path);
- failed = TRUE;
- }
- } else {
- (void)close(index->maildir_lock_fd);
- }
- index->maildir_lock_fd = -1;
-
- if (failed)
- (void)unlink(temp_path);
- return !failed;
-}
+++ /dev/null
-#ifndef __MAILDIR_UIDLIST_H
-#define __MAILDIR_UIDLIST_H
-
-#define INDEX_IS_UIDLIST_LOCKED(index) \
- ((index)->maildir_lock_fd != -1)
-
-#define MAILDIR_UIDLIST_NAME "dovecot-uidlist"
-
-struct maildir_uidlist {
- struct mail_index *index;
- char *fname;
- struct istream *input;
-
- unsigned int uid_validity, next_uid, last_read_uid;
-};
-
-struct maildir_uidlist_rec {
- unsigned int uid;
- const char *filename;
-};
-
-int maildir_uidlist_try_lock(struct mail_index *index);
-void maildir_uidlist_unlock(struct mail_index *index);
-int maildir_uidlist_rewrite(struct mail_index *index, time_t *mtime);
-
-struct maildir_uidlist *maildir_uidlist_open(struct mail_index *index);
-void maildir_uidlist_close(struct maildir_uidlist *uidlist);
-
-/* Returns -1 if error, 0 if end of file or 1 if found.
- uid_rec.uid is also set to 0 at EOF. This function does sanity checks so
- you can be sure that uid_rec.uid is always growing and smaller than
- uidlist->next_uid. */
-int maildir_uidlist_next(struct maildir_uidlist *uidlist,
- struct maildir_uidlist_rec *uid_rec);
-
-/* Try to update cur/ stamp in */
-int maildir_uidlist_update_cur_stamp(struct maildir_uidlist *uidlist,
- time_t stamp);
-
-#endif
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "hash.h"
-#include "ioloop.h"
-#include "maildir-index.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-
-#include <stdio.h>
-#include <sys/stat.h>
-
-struct update_flags_ctx {
- const char *new_fname;
- int found;
-
- enum modify_type modify_type;
- enum mail_flags flags;
-};
-
-static int update_filename(struct mail_index *index,
- struct mail_index_record *rec)
-{
- const char *old_fname, *old_path, *new_fname, *new_path;
- enum mail_index_record_flag flags;
-
- old_fname = maildir_get_location(index, rec, NULL);
- if (old_fname == NULL)
- return -1;
-
- flags = mail_cache_get_index_flags(index->cache, rec);
-
- old_path = t_strconcat(index->mailbox_path,
- (flags & MAIL_INDEX_FLAG_MAILDIR_NEW) != 0 ?
- "/new/" : "/cur/", old_fname, NULL);
-
- new_fname = maildir_filename_set_flags(old_fname, rec->msg_flags);
- new_path = t_strconcat(index->mailbox_path, "/cur/", new_fname, NULL);
-
- if (strcmp(old_path, new_path) == 0 ||
- rename(old_path, new_path) == 0) {
- flags &= ~(MAIL_INDEX_FLAG_DIRTY | MAIL_INDEX_FLAG_MAILDIR_NEW);
- if (!mail_cache_update_index_flags(index->cache, rec, flags))
- return -1;
- return 1;
- } else {
- if (errno != ENOENT && errno != EACCES &&
- !ENOSPACE(errno)) {
- index_set_error(index,
- "rename(%s, %s) failed: %m",
- old_path, new_path);
- return -1;
- }
- return 0;
- }
-}
-
-int maildir_try_flush_dirty_flags(struct mail_index *index, int force)
-{
- struct mail_index_record *rec;
- int ret, dirty = FALSE;
-
- if (index->next_dirty_flags_flush == 0 ||
- (ioloop_time < index->next_dirty_flags_flush && !force))
- return TRUE;
-
- if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
-
- ret = mail_cache_lock(index->cache, !force);
- if (ret <= 0)
- return ret == 0;
- mail_cache_unlock_later(index->cache);
-
- rec = index->lookup(index, 1);
- while (rec != NULL) {
- if ((mail_cache_get_index_flags(index->cache, rec) &
- MAIL_INDEX_FLAG_DIRTY) != 0) {
- ret = update_filename(index, rec);
- if (ret < 0)
- break;
- if (ret == 0)
- dirty = TRUE;
- }
-
- rec = index->next(index, rec);
- }
-
- if (ret < 0)
- return FALSE;
-
- if (!dirty) {
- index->header->flags &= ~MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES;
- index->next_dirty_flags_flush = 0;
- } else {
- index->next_dirty_flags_flush =
- ioloop_time + MAILDIR_DIRTY_FLUSH_TIMEOUT;
- }
-
- return TRUE;
-}
-
-static int do_rename(struct mail_index *index, const char *path, void *context)
-{
- struct update_flags_ctx *ctx = context;
- const char *fname, *new_path;
- enum mail_flags old_flags, new_flags;
- int new_dir;
-
- old_flags = maildir_filename_get_flags(path, 0);
- switch (ctx->modify_type) {
- case MODIFY_ADD:
- new_flags = old_flags | ctx->flags;
- break;
- case MODIFY_REMOVE:
- new_flags = old_flags & ~ctx->flags;
- break;
- case MODIFY_REPLACE:
- new_flags = ctx->flags |
- (old_flags & index->private_flags_mask);
- break;
- default:
- new_flags = 0;
- i_unreached();
- }
-
- fname = strrchr(path, '/');
- ctx->new_fname = maildir_filename_set_flags(fname != NULL ?
- fname+1 : path, new_flags);
-
- if (old_flags == new_flags) {
- /* it's what we wanted. verify that the file exists, but
- only if something actually could have changed
- (ie. do nothing with private flag changes in shared
- mailboxes). */
- struct stat st;
-
- if (ctx->flags != 0) {
- if (stat(path, &st) < 0) {
- if (errno == ENOENT)
- return 0;
- index_file_set_syscall_error(index, path,
- "stat()");
- return -1;
- }
- }
- ctx->found = TRUE;
- return 1;
- }
-
- new_dir = fname != NULL && path + 4 <= fname &&
- strncmp(fname-4, "/new", 4) == 0;
- if (new_dir) {
- /* move from new/ to cur/ */
- new_path = t_strconcat(t_strdup_until(path, fname-4),
- "/cur/", ctx->new_fname, NULL);
- } else {
- new_path = maildir_filename_set_flags(path, new_flags);
- }
-
- if (rename(path, new_path) < 0) {
- if (errno == ENOENT)
- return 0;
-
- if (ENOSPACE(errno)) {
- index->nodiskspace = TRUE;
- return 1;
- }
-
- if (errno == EACCES) {
- index->mailbox_readonly = TRUE;
- return 1;
- }
-
- index_set_error(index, "rename(%s, %s) failed: %m",
- path, new_path);
- return -1;
- }
-
- if (index->maildir_keep_new && new_dir) {
- /* looks like we have some more space again, see if we could
- move mails from new/ to cur/ again */
- index->maildir_keep_new = FALSE;
- }
-
- /* cur/ was updated, set it dirty-synced */
- index->sync_stamp = ioloop_time;
- index->sync_dirty_stamp = ioloop_time;
- ctx->found = TRUE;
- return 1;
-}
-
-int maildir_index_update_flags(struct mail_index *index,
- struct mail_index_record *rec, unsigned int seq,
- enum modify_type modify_type,
- enum mail_flags flags, int external_change)
-{
- struct update_flags_ctx ctx;
- enum mail_index_record_flag index_flags;
-
- memset(&ctx, 0, sizeof(ctx));
- ctx.modify_type = modify_type;
- ctx.flags = flags & ~index->private_flags_mask;
-
- t_push();
- if (!maildir_file_do(index, rec, do_rename, &ctx)) {
- t_pop();
- return FALSE;
- }
-
- if (!ctx.found) {
- /* we couldn't actually rename() the file now.
- leave it's flags dirty so they get changed later. */
- index_flags = mail_cache_get_index_flags(index->cache, rec);
- if ((index_flags & MAIL_INDEX_FLAG_DIRTY) == 0) {
- if (mail_cache_lock(index->cache, FALSE) <= 0)
- return FALSE;
- mail_cache_unlock_later(index->cache);
-
- index_flags |= MAIL_INDEX_FLAG_DIRTY;
- mail_cache_update_index_flags(index->cache, rec,
- index_flags);
-
- index->header->flags |=
- MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES;
- }
-
- index->next_dirty_flags_flush =
- ioloop_time + MAILDIR_DIRTY_FLUSH_TIMEOUT;
- } else if (ctx.new_fname != NULL) {
- maildir_index_update_filename(index, rec->uid,
- ctx.new_fname, FALSE);
- }
- t_pop();
-
- return mail_index_update_flags(index, rec, seq,
- modify_type, flags, external_change);
-}
+++ /dev/null
-/* Copyright (C) 2003 Timo Sirainen */
-
-#include "lib.h"
-#include "buffer.h"
-#include "message-parser.h"
-#include "istream-internal.h"
-#include "mbox-index.h"
-
-struct mbox_istream {
- struct _istream istream;
-
- struct istream *input;
-
- buffer_t *headers;
- uoff_t v_header_size, body_offset, body_size;
-};
-
-static void _close(struct _iostream *stream __attr_unused__)
-{
-}
-
-static void _destroy(struct _iostream *stream)
-{
- struct mbox_istream *mstream = (struct mbox_istream *) stream;
-
- i_stream_unref(mstream->input);
- buffer_free(mstream->headers);
-}
-
-static void _set_max_buffer_size(struct _iostream *stream, size_t max_size)
-{
- struct mbox_istream *mstream = (struct mbox_istream *) stream;
-
- i_stream_set_max_buffer_size(mstream->input, max_size);
-}
-
-static void _set_blocking(struct _iostream *stream, int timeout_msecs,
- void (*timeout_cb)(void *), void *context)
-{
- struct mbox_istream *mstream = (struct mbox_istream *) stream;
-
- i_stream_set_blocking(mstream->input, timeout_msecs,
- timeout_cb, context);
-}
-
-static ssize_t _read(struct _istream *stream)
-{
- struct mbox_istream *mstream = (struct mbox_istream *) stream;
- ssize_t ret;
- size_t pos;
- uoff_t offset;
-
- if (stream->istream.v_offset < mstream->v_header_size) {
- /* we don't support mixing headers and body.
- it shouldn't be needed. */
- return -2;
- }
-
- offset = stream->istream.v_offset - mstream->v_header_size;
- if (mstream->input->v_offset != offset)
- i_stream_seek(mstream->input, offset);
-
- ret = i_stream_read(mstream->input);
-
- stream->pos -= stream->skip;
- stream->skip = 0;
- stream->buffer = i_stream_get_data(mstream->input, &pos);
-
- ret = pos <= stream->pos ? -1 :
- (ssize_t) (pos - stream->pos);
- mstream->istream.pos = pos;
- return ret;
-}
-
-static void _seek(struct _istream *stream, uoff_t v_offset)
-{
- struct mbox_istream *mstream = (struct mbox_istream *) stream;
-
- stream->istream.v_offset = v_offset;
- if (v_offset < mstream->v_header_size) {
- /* still in headers */
- stream->skip = v_offset;
- stream->pos = mstream->v_header_size;
- stream->buffer = buffer_get_data(mstream->headers, NULL);
- } else {
- /* body - use our real input stream */
- stream->skip = stream->pos = 0;
- stream->buffer = NULL;
- }
-}
-
-struct istream *i_stream_create_mbox(pool_t pool, struct istream *input,
- uoff_t offset, uoff_t body_size)
-{
- struct mbox_istream *mstream;
- struct istream *hdr_input;
-
- mstream = p_new(pool, struct mbox_istream, 1);
- mstream->body_size = body_size;
-
- if (body_size == 0) {
- /* possibly broken message, find the next From-line
- and make sure header parser won't pass it. */
- mbox_skip_header(input);
- hdr_input = i_stream_create_limit(pool, input,
- 0, input->v_offset);
- } else {
- hdr_input = input;
- i_stream_ref(input);
- }
-
- mstream->headers = buffer_create_dynamic(default_pool,
- 8192, (size_t)-1);
- i_stream_seek(hdr_input, offset);
- mbox_read_headers(hdr_input, mstream->headers);
- mstream->v_header_size = buffer_get_used_size(mstream->headers);
- mstream->body_offset = hdr_input->v_offset;
- i_stream_unref(hdr_input);
-
- mstream->input = i_stream_create_limit(pool, input,
- mstream->body_offset, body_size);
-
- mstream->istream.buffer = buffer_get_data(mstream->headers, NULL);
- mstream->istream.pos = mstream->v_header_size;
-
- mstream->istream.iostream.close = _close;
- mstream->istream.iostream.destroy = _destroy;
- mstream->istream.iostream.set_max_buffer_size = _set_max_buffer_size;
- mstream->istream.iostream.set_blocking = _set_blocking;
-
- mstream->istream.read = _read;
- mstream->istream.seek = _seek;
-
- return _i_stream_create(&mstream->istream, pool, -1,
- input->real_stream->abs_start_offset);
-}
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "ioloop.h"
-#include "istream.h"
-#include "hex-binary.h"
-#include "md5.h"
-#include "mbox-index.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-
-static int mbox_index_append_next(struct mail_index *index,
- struct mail_cache_transaction_ctx *trans_ctx,
- struct istream *input)
-{
- struct mail_index_record *rec;
- struct mbox_header_context ctx;
- struct istream *hdr_stream;
- enum mail_index_record_flag index_flags;
- time_t received_date;
- uoff_t hdr_offset, body_offset, end_offset;
- const unsigned char *data;
- unsigned char md5_digest[16];
- size_t size, pos;
- int dirty, save_md5 = FALSE;
-
- /* get the From-line */
- pos = 0;
- while (i_stream_read_data(input, &data, &size, pos) > 0) {
- for (; pos < size; pos++) {
- if (data[pos] == '\n')
- break;
- }
-
- if (pos < size)
- break;
- }
-
- if (size == 0)
- return -2;
-
- if (pos == size || size <= 5 || memcmp(data, "From ", 5) != 0) {
- /* a) no \n found, or line too long
- b) not a From-line */
- index_set_error(index, "Error indexing mbox file %s: "
- "From-line not found where expected",
- index->mailbox_path);
- index->set_flags |= MAIL_INDEX_HDR_FLAG_FSCK;
- return -1;
- }
-
- /* parse the From-line */
- received_date = mbox_from_parse_date(data + 5, size - 5);
- if (received_date == (time_t)-1)
- received_date = ioloop_time;
-
- i_stream_skip(input, pos+1);
- hdr_offset = input->v_offset;
-
- /* now, find the end of header. also stops at "\nFrom " if it's
- found (broken messages) */
- mbox_skip_header(input);
- body_offset = input->v_offset;
-
- index_flags = 0;
-
- /* parse the header and cache wanted fields. get the message flags
- from Status and X-Status fields. temporarily limit the stream length
- so the message body is parsed properly.
-
- the stream length limit is raised again by mbox_header_cb after
- reading the headers. it uses Content-Length if available or finds
- the next From-line. */
- mbox_header_init_context(&ctx, index, input);
-
- hdr_stream = i_stream_create_limit(default_pool, input,
- hdr_offset,
- body_offset - hdr_offset);
- i_stream_seek(hdr_stream, 0);
- message_parse_header(NULL, hdr_stream, NULL, mbox_header_cb, &ctx);
- i_stream_unref(hdr_stream);
-
- dirty = FALSE;
-
- /* try Content-Length */
- end_offset = body_offset + ctx.content_length;
- if (ctx.content_length == (uoff_t)-1 ||
- !mbox_verify_end_of_body(input, end_offset)) {
- /* failed, search for From-line */
- if (ctx.content_length != (uoff_t)-1) {
- /* broken, rewrite it */
- dirty = TRUE;
- }
-
- i_stream_seek(input, body_offset);
- mbox_skip_message(input);
- ctx.content_length = input->v_offset - body_offset;
- }
-
- if (index->header->messages_count == 0 &&
- ctx.uid_validity != index->header->uid_validity) {
- /* UID validity is different */
- if (ctx.uid_validity != 0) {
- /* change it in index */
- index->header->uid_validity = ctx.uid_validity;
- index->header->next_uid = 1;
- index->header->last_nonrecent_uid = 0;
- index->inconsistent = TRUE;
- } else if (!index->mailbox_readonly) {
- /* we have to write it to mbox */
- if (index->mbox_lock_type != MAIL_LOCK_EXCLUSIVE) {
- /* try again */
- return 0;
- }
-
- dirty = TRUE;
- }
- }
-
- if (ctx.uid >= index->header->next_uid) {
- /* X-UID header looks ok */
- index->header->next_uid = ctx.uid;
- } else if (!index->mailbox_readonly) {
- /* Write X-UID for it */
- dirty = TRUE;
- } else {
- /* save MD5 */
- save_md5 = TRUE;
- }
-
- if (dirty && !index->mailbox_readonly) {
- if (index->mbox_lock_type != MAIL_LOCK_EXCLUSIVE) {
- /* try again */
- return 0;
- }
-
- index->header->flags |= MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES;
- index_flags |= MAIL_INDEX_FLAG_DIRTY;
- }
-
- /* add message to index */
- rec = index->append(index);
- if (rec == NULL)
- return -1;
-
- /* save message flags */
- rec->msg_flags = ctx.flags;
- mail_index_mark_flag_changes(index, rec, 0, rec->msg_flags);
-
- if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_INDEX_FLAGS,
- &index_flags, sizeof(index_flags)))
- return -1;
-
- /* location offset = beginning of headers in message */
- if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_LOCATION_OFFSET,
- &hdr_offset, sizeof(hdr_offset)))
- return -1;
-
- if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_RECEIVED_DATE,
- &received_date, sizeof(received_date)))
- return -1;
-
- if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_PHYSICAL_BODY_SIZE,
- &ctx.content_length, sizeof(ctx.content_length)))
- return -1;
-
- if (save_md5) {
- md5_final(&ctx.md5, md5_digest);
-
- if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_MD5,
- md5_digest, sizeof(md5_digest)))
- return -1;
- }
-
- return 1;
-}
-
-int mbox_index_append_stream(struct mail_index *index, struct istream *input)
-{
- struct mail_cache_transaction_ctx *trans_ctx;
- uoff_t offset;
- int ret;
-
- if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
- return -1;
-
- if (mail_cache_transaction_begin(index->cache, TRUE, &trans_ctx) <= 0)
- return -1;
-
- do {
- offset = input->v_offset;
- if (input->v_offset != 0) {
- /* we're at the [\r]\n before the From-line,
- skip it */
- if (!mbox_skip_crlf(input)) {
- index_set_error(index,
- "Error indexing mbox file %s: "
- "LF not found where expected",
- index->mailbox_path);
-
- index->set_flags |= MAIL_INDEX_HDR_FLAG_FSCK;
- ret = -1;
- break;
- }
- }
-
- t_push();
- ret = mbox_index_append_next(index, trans_ctx, input);
- t_pop();
-
- if (ret == -2) {
- /* EOF */
- ret = 1;
- break;
- }
-
- if (ret == 0) {
- /* we want to rescan this message with exclusive
- locking */
- i_stream_seek(input, offset);
- }
- } while (ret > 0);
-
- if (ret >= 0 && index->mbox_lock_type == MAIL_LOCK_EXCLUSIVE) {
- /* Write missing X-IMAPbase and new/changed X-UID headers */
- if (!mbox_index_rewrite(index))
- ret = -1;
- }
-
- if (ret >= 0) {
- if (!mail_cache_transaction_commit(trans_ctx))
- ret = -1;
- }
- if (!mail_cache_transaction_end(trans_ctx))
- ret = -1;
-
- return ret;
-}
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "str.h"
-#include "utc-mktime.h"
-#include "mbox-index.h"
-
-#include <time.h>
-#include <ctype.h>
-
-static const char *weekdays[] = {
- "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"
-};
-
-static const char *months[] = {
- "Jan", "Feb", "Mar", "Apr", "May", "Jun",
- "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
-};
-
-time_t mbox_from_parse_date(const unsigned char *msg, size_t size)
-{
- const unsigned char *msg_end;
- struct tm tm;
- int i, timezone;
- time_t t;
-
- /* <sender> <date> <moreinfo> */
- msg_end = msg + size;
-
- /* skip sender */
- while (msg < msg_end && *msg != ' ') {
- if (*msg == '\r' || *msg == '\n')
- return (time_t)-1;
- msg++;
- }
- while (msg < msg_end && *msg == ' ') msg++;
-
- /* next 24 chars should be in the date in asctime() format, eg.
- "Thu Nov 29 22:33:52 2001 +0300"
-
- Some also include named timezone, which we ignore:
-
- "Thu Nov 29 22:33:52 EEST 2001"
- */
- if (msg+24 > msg_end)
- return (time_t)-1;
-
- memset(&tm, 0, sizeof(tm));
-
- /* skip weekday */
- msg += 4;
-
- /* month */
- for (i = 0; i < 12; i++) {
- if (memcasecmp(months[i], msg, 3) == 0) {
- tm.tm_mon = i;
- break;
- }
- }
-
- if (i == 12 && memcmp(msg, "???", 3) == 0) {
- /* just a hack to parse one special mbox I have :) */
- i = 0;
- }
-
- if (i == 12 || msg[3] != ' ')
- return (time_t)-1;
- msg += 4;
-
- /* day */
- if (msg[0] == ' ') {
- if (!i_isdigit(msg[1]) || msg[2] != ' ')
- return (time_t)-1;
- tm.tm_mday = msg[1]-'0';
- } else {
- if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ' ')
- return (time_t)-1;
- tm.tm_mday = (msg[0]-'0') * 10 + (msg[1]-'0');
- }
- if (tm.tm_mday == 0)
- tm.tm_mday = 1;
- msg += 3;
-
- /* hour */
- if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ':')
- return (time_t)-1;
- tm.tm_hour = (msg[0]-'0') * 10 + (msg[1]-'0');
- msg += 3;
-
- /* minute */
- if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ':')
- return (time_t)-1;
- tm.tm_min = (msg[0]-'0') * 10 + (msg[1]-'0');
- msg += 3;
-
- /* second */
- if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ' ')
- return (time_t)-1;
- tm.tm_sec = (msg[0]-'0') * 10 + (msg[1]-'0');
- msg += 3;
-
- /* optional named timezone */
- if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) ||
- !i_isdigit(msg[2]) || !i_isdigit(msg[3])) {
- /* skip to next space */
- while (msg < msg_end && *msg != ' ') {
- if (*msg == '\r' || *msg == '\n')
- return (time_t)-1;
- msg++;
- }
- if (msg+5 > msg_end)
- return (time_t)-1;
- msg++;
- }
-
- /* year */
- if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) ||
- !i_isdigit(msg[2]) || !i_isdigit(msg[3]))
- return (time_t)-1;
-
- tm.tm_year = (msg[0]-'0') * 1000 + (msg[1]-'0') * 100 +
- (msg[2]-'0') * 10 + (msg[3]-'0') - 1900;
- msg += 4;
-
- tm.tm_isdst = -1;
- if (msg[0] == ' ' && (msg[1] == '-' || msg[1] == '+') &&
- i_isdigit(msg[2]) && i_isdigit(msg[3]) &&
- i_isdigit(msg[4]) && i_isdigit(msg[5])) {
- timezone = (msg[2]-'0') * 1000 + (msg[3]-'0') * 100 +
- (msg[4]-'0') * 10 +(msg[5]-'0');
- if (msg[1] == '-') timezone = -timezone;
-
- t = utc_mktime(&tm);
- if (t == (time_t)-1)
- return (time_t)-1;
-
- t -= timezone * 60;
- return t;
- } else {
- /* assume local timezone */
- return mktime(&tm);
- }
-}
-
-const char *mbox_from_create(const char *sender, time_t time)
-{
- string_t *str;
- struct tm *tm;
- int year;
-
- str = t_str_new(256);
- str_append(str, "From ");
- str_append(str, sender);
- str_append(str, " ");
-
- /* we could use simply asctime(), but i18n etc. may break it.
- Example: "Thu Nov 29 22:33:52 2001" */
- tm = localtime(&time);
-
- /* week day */
- str_append(str, weekdays[tm->tm_wday]);
- str_append_c(str, ' ');
-
- /* month */
- str_append(str, months[tm->tm_mon]);
- str_append_c(str, ' ');
-
- /* day */
- str_append_c(str, (tm->tm_mday / 10) + '0');
- str_append_c(str, (tm->tm_mday % 10) + '0');
- str_append_c(str, ' ');
-
- /* hour */
- str_append_c(str, (tm->tm_hour / 10) + '0');
- str_append_c(str, (tm->tm_hour % 10) + '0');
- str_append_c(str, ':');
-
- /* minute */
- str_append_c(str, (tm->tm_min / 10) + '0');
- str_append_c(str, (tm->tm_min % 10) + '0');
- str_append_c(str, ':');
-
- /* second */
- str_append_c(str, (tm->tm_sec / 10) + '0');
- str_append_c(str, (tm->tm_sec % 10) + '0');
- str_append_c(str, ' ');
-
- /* year */
- year = tm->tm_year + 1900;
- str_append_c(str, (year / 1000) + '0');
- str_append_c(str, ((year / 100) % 10) + '0');
- str_append_c(str, ((year / 10) % 10) + '0');
- str_append_c(str, (year % 10) + '0');
-
- str_append_c(str, '\n');
- return str_c(str);
-}
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "buffer.h"
-#include "istream.h"
-#include "message-part-serialize.h"
-#include "mbox-index.h"
-#include "mbox-lock.h"
-#include "mail-index-util.h"
-#include "mail-custom-flags.h"
-#include "mail-cache.h"
-
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-
-/* Don't try reading more custom flags than this. */
-#define MAX_CUSTOM_FLAGS 1024
-
-extern struct mail_index mbox_index;
-
-int mbox_set_syscall_error(struct mail_index *index, const char *function)
-{
- i_assert(function != NULL);
-
- index_set_error(index, "%s failed with mbox file %s: %m",
- function, index->mailbox_path);
- return FALSE;
-}
-
-int mbox_file_open(struct mail_index *index)
-{
- struct stat st;
- int fd;
-
- i_assert(index->mbox_fd == -1);
-
- fd = open(index->mailbox_path, index->mailbox_readonly ?
- O_RDONLY : O_RDWR);
- if (fd == -1) {
- mbox_set_syscall_error(index, "open()");
- return FALSE;
- }
-
- if (fstat(fd, &st) < 0) {
- mbox_set_syscall_error(index, "fstat()");
- (void)close(fd);
- return FALSE;
- }
-
- index->mbox_fd = fd;
- index->mbox_dev = st.st_dev;
- index->mbox_ino = st.st_ino;
- return TRUE;
-}
-
-struct istream *mbox_get_stream(struct mail_index *index,
- enum mail_lock_type lock_type)
-{
- switch (lock_type) {
- case MAIL_LOCK_SHARED:
- case MAIL_LOCK_EXCLUSIVE:
- /* don't drop exclusive lock, it may be there for a reason */
- if (index->mbox_lock_type != MAIL_LOCK_EXCLUSIVE) {
- if (!mbox_lock(index, lock_type))
- return NULL;
- }
- break;
- default:
- if (index->mbox_fd == -1) {
- if (!mbox_file_open(index))
- return NULL;
- }
- break;
- }
-
- if (index->mbox_stream == NULL) {
- if (index->mail_read_mmaped) {
- index->mbox_stream =
- i_stream_create_mmap(index->mbox_fd,
- default_pool,
- MAIL_MMAP_BLOCK_SIZE,
- 0, 0, FALSE);
- } else {
- index->mbox_stream =
- i_stream_create_file(index->mbox_fd,
- default_pool,
- MAIL_READ_BLOCK_SIZE,
- FALSE);
- }
- }
-
- i_stream_seek(index->mbox_stream, 0);
- i_stream_ref(index->mbox_stream);
- return index->mbox_stream;
-}
-
-void mbox_file_close_stream(struct mail_index *index)
-{
- if (index->mbox_stream != NULL) {
- i_stream_close(index->mbox_stream);
- i_stream_unref(index->mbox_stream);
- index->mbox_stream = NULL;
- }
-}
-
-void mbox_file_close_fd(struct mail_index *index)
-{
- mbox_file_close_stream(index);
-
- if (index->mbox_fd != -1) {
- if (close(index->mbox_fd) < 0)
- i_error("close(mbox) failed: %m");
- index->mbox_fd = -1;
- }
-}
-
-void mbox_header_init_context(struct mbox_header_context *ctx,
- struct mail_index *index,
- struct istream *input)
-{
- memset(ctx, 0, sizeof(struct mbox_header_context));
- md5_init(&ctx->md5);
-
- ctx->index = index;
- ctx->input = input;
- ctx->custom_flags = mail_custom_flags_list_get(index->custom_flags);
- ctx->content_length = (uoff_t)-1;
-}
-
-static enum mail_flags
-mbox_get_status_flags(const unsigned char *value, size_t len)
-{
- enum mail_flags flags;
- size_t i;
-
- flags = 0;
- for (i = 0; i < len; i++) {
- switch (value[i]) {
- case 'A':
- flags |= MAIL_ANSWERED;
- break;
- case 'F':
- flags |= MAIL_FLAGGED;
- break;
- case 'T':
- flags |= MAIL_DRAFT;
- break;
- case 'R':
- flags |= MAIL_SEEN;
- break;
- case 'D':
- flags |= MAIL_DELETED;
- break;
- }
- }
-
- return flags;
-}
-
-static void mbox_update_custom_flags(const unsigned char *value __attr_unused__,
- size_t len __attr_unused__,
- int index, void *context)
-{
- enum mail_flags *flags = context;
-
- if (index >= 0)
- *flags |= 1 << (index + MAIL_CUSTOM_FLAG_1_BIT);
-}
-
-static enum mail_flags
-mbox_get_keyword_flags(const unsigned char *value, size_t len,
- const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT])
-{
- enum mail_flags flags;
-
- flags = 0;
- mbox_keywords_parse(value, len, custom_flags,
- mbox_update_custom_flags, &flags);
- return flags;
-}
-
-static void mbox_parse_imapbase(const unsigned char *value, size_t len,
- struct mbox_header_context *ctx)
-{
- const char *flag, *str;
- char *end;
- buffer_t *buf;
- size_t pos, start;
- enum mail_flags flags;
- unsigned int count;
- int ret;
-
- t_push();
-
- /* <uid validity> <last uid> */
- str = t_strndup(value, len);
- ctx->uid_validity = strtoul(str, &end, 10);
- ctx->uid_last = strtoul(end, &end, 10);
- pos = end - str;
-
- while (pos < len && value[pos] == ' ')
- pos++;
-
- if (pos == len) {
- t_pop();
- return;
- }
-
- /* we're at the 3rd field now, which begins the list of custom flags */
- buf = buffer_create_dynamic(pool_datastack_create(),
- MAIL_CUSTOM_FLAGS_COUNT *
- sizeof(const char *),
- MAX_CUSTOM_FLAGS * sizeof(const char *));
- for (start = pos; ; pos++) {
- if (pos == len || value[pos] == ' ' || value[pos] == '\t') {
- if (start != pos) {
- flag = t_strdup_until(value+start, value+pos);
- if (buffer_append(buf, &flag,
- sizeof(flag)) == 0)
- break;
- }
- start = pos+1;
-
- if (pos == len)
- break;
- }
- }
-
- flags = MAIL_CUSTOM_FLAGS_MASK;
- count = buffer_get_used_size(buf) / sizeof(const char *);
- ret = mail_custom_flags_fix_list(ctx->index->custom_flags, &flags,
- buffer_free_without_data(buf), count);
-
- t_pop();
-}
-
-void mbox_header_cb(struct message_part *part __attr_unused__,
- struct message_header_line *hdr, void *context)
-{
- struct mbox_header_context *ctx = context;
- size_t i;
- int fixed = FALSE;
-
- if (hdr == NULL || hdr->eoh)
- return;
-
- /* Pretty much copy&pasted from popa3d by Solar Designer */
- switch (*hdr->name) {
- case 'R':
- case 'r':
- if (!ctx->received &&
- strcasecmp(hdr->name, "Received") == 0) {
- /* get only the first received-header */
- fixed = TRUE;
- if (!hdr->continues)
- ctx->received = TRUE;
- }
- break;
-
- case 'C':
- case 'c':
- if (strcasecmp(hdr->name, "Content-Length") == 0) {
- /* manual parsing, so we can deal with uoff_t */
- ctx->content_length = 0;
- for (i = 0; i < hdr->value_len; i++) {
- if (hdr->value[i] < '0' ||
- hdr->value[i] > '9') {
- /* invalid */
- ctx->content_length = 0;
- break;
- }
-
- ctx->content_length = ctx->content_length * 10 +
- (hdr->value[i] - '0');
- }
- }
- break;
-
- case 'D':
- case 'd':
- if (strcasecmp(hdr->name, "Delivered-To") == 0)
- fixed = TRUE;
- else if (!ctx->received && strcasecmp(hdr->name, "Date") == 0) {
- /* Received-header contains date too,
- and more trusted one */
- fixed = TRUE;
- }
- break;
-
- case 'M':
- case 'm':
- if (!ctx->received &&
- strcasecmp(hdr->name, "Message-ID") == 0) {
- /* Received-header contains unique ID too,
- and more trusted one */
- fixed = TRUE;
- }
- break;
-
- case 'S':
- case 's':
- if (strcasecmp(hdr->name, "Status") == 0) {
- /* update message flags */
- ctx->flags |= mbox_get_status_flags(hdr->value,
- hdr->value_len);
- }
- break;
-
- case 'X':
- case 'x':
- if (strcasecmp(hdr->name, "X-Delivery-ID:") == 0) {
- /* Let the local delivery agent help generate unique
- ID's but don't blindly trust this header alone as
- it could just as easily come from the remote. */
- fixed = TRUE;
- } else if (strcasecmp(hdr->name, "X-UID") == 0) {
- ctx->uid = 0;
- for (i = 0; i < hdr->value_len; i++) {
- if (hdr->value[i] < '0' ||
- hdr->value[i] > '9')
- break;
- ctx->uid = ctx->uid * 10 + (hdr->value[i]-'0');
- }
- } else if (strcasecmp(hdr->name, "X-Status") == 0) {
- /* update message flags */
- ctx->flags |= mbox_get_status_flags(hdr->value,
- hdr->value_len);
- } else if (strcasecmp(hdr->name, "X-Keywords") == 0) {
- /* update custom message flags */
- ctx->flags |= mbox_get_keyword_flags(hdr->value,
- hdr->value_len,
- ctx->custom_flags);
- } else if (strcasecmp(hdr->name, "X-IMAPbase") == 0) {
- if (hdr->continues) {
- hdr->use_full_value = TRUE;
- break;
- }
- mbox_parse_imapbase(hdr->full_value,
- hdr->full_value_len, ctx);
- }
- break;
- }
-
- if (fixed)
- md5_update(&ctx->md5, hdr->value, hdr->value_len);
-}
-
-void mbox_keywords_parse(const unsigned char *value, size_t len,
- const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT],
- void (*func)(const unsigned char *, size_t,
- int, void *),
- void *context)
-{
- size_t custom_len[MAIL_CUSTOM_FLAGS_COUNT];
- size_t item_len;
- int i;
-
- /* the value is often empty, so check that first */
- while (len > 0 && IS_LWSP(*value)) {
- value++;
- len--;
- }
-
- if (len == 0)
- return;
-
- for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
- custom_len[i] = custom_flags[i] != NULL ?
- strlen(custom_flags[i]) : 0;
- }
-
- for (;;) {
- /* skip whitespace */
- while (len > 0 && IS_LWSP(*value)) {
- value++;
- len--;
- }
-
- if (len == 0)
- break;
-
- /* find the length of the item */
- for (item_len = 0; item_len < len; item_len++) {
- if (IS_LWSP(value[item_len]))
- break;
- }
-
- /* check if it's found */
- for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
- if (custom_len[i] == item_len &&
- memcasecmp(custom_flags[i], value, item_len) == 0)
- break;
- }
-
- if (i == MAIL_CUSTOM_FLAGS_COUNT)
- i = -1;
-
- func(value, item_len, i, context);
-
- value += item_len;
- len -= item_len;
- }
-}
-
-int mbox_skip_crlf(struct istream *input)
-{
- const unsigned char *data;
- size_t size, pos;
-
- pos = 0;
- while (i_stream_read_data(input, &data, &size, pos) > 0) {
- if (pos == 0) {
- if (data[0] == '\n') {
- i_stream_skip(input, 1);
- return TRUE;
- }
- if (data[0] != '\r')
- return FALSE;
-
- pos++;
- }
-
- if (size > 1 && pos == 1) {
- if (data[1] != '\n')
- return FALSE;
-
- i_stream_skip(input, 2);
- return TRUE;
- }
- }
-
- /* end of file */
- return TRUE;
-}
-
-void mbox_skip_empty_lines(struct istream *input)
-{
- const unsigned char *data;
- size_t i, size;
-
- /* skip empty lines at beginning */
- while (i_stream_read_data(input, &data, &size, 0) > 0) {
- for (i = 0; i < size; i++) {
- if (data[i] != '\r' && data[i] != '\n')
- break;
- }
-
- i_stream_skip(input, i);
-
- if (i < size)
- break;
- }
-}
-
-static int mbox_is_valid_from(struct istream *input, size_t startpos)
-{
- const unsigned char *msg;
- size_t i, size;
-
- i = startpos;
- while (i_stream_read_data(input, &msg, &size, i) > 0) {
- for (; i < size; i++) {
- if (msg[i] == '\n') {
- msg += startpos;
- i -= startpos;
- return mbox_from_parse_date(msg, size) !=
- (time_t)-1;
- }
- }
- }
-
- return FALSE;
-}
-
-static void mbox_skip_forward(struct istream *input, int header)
-{
- const unsigned char *msg;
- size_t i, size, startpos, eoh;
- int lastmsg, state, new_state;
-
- /* read until "[\r]\nFrom " is found. assume '\n' at beginning of
- buffer */
- startpos = i = 0; eoh = 0; lastmsg = TRUE;
- state = '\n';
- while (i_stream_read_data(input, &msg, &size, startpos) > 0) {
- for (i = startpos; i < size; i++) {
- new_state = 0;
- switch (state) {
- case '\n':
- if (msg[i] == 'F')
- new_state = 'F';
- else if (header) {
- if (msg[i] == '\n') {
- /* \n\n, but if we have
- 0-byte message body the
- following \n may belong
- to "From "-line */
- eoh = i+1;
- header = FALSE;
- new_state = '\n';
- } else if (msg[i] == '\r') {
- /* possibly \n\r\n */
- new_state = '\r';
- }
- }
- break;
- case '\r':
- if (msg[i] == '\n') {
- /* \n\r\n */
- eoh = i+1;
- header = FALSE;
- new_state = '\n';
- }
- break;
- case 'F':
- if (msg[i] == 'r')
- new_state = 'r';
- break;
- case 'r':
- if (msg[i] == 'o')
- new_state = 'o';
- break;
- case 'o':
- if (msg[i] == 'm')
- new_state = 'm';
- break;
- case 'm':
- if (msg[i] == ' ') {
- int valid;
-
- valid = mbox_is_valid_from(input, i+1);
-
- /* we may have trashed msg above,
- get it again */
- msg = i_stream_get_data(input, &size);
-
- if (valid) {
- /* Go back "From" */
- i -= 4;
-
- /* Go back \n, unless we're at
- beginning of buffer */
- if (i > 0)
- i--;
-
- /* Go back \r if it's there */
- if (i > 0 && msg[i-1] == '\r')
- i--;
-
- i_stream_skip(input, i);
- return;
- }
- }
- break;
- }
-
- if (new_state != 0)
- state = new_state;
- else if (eoh == 0)
- state = msg[i] == '\n' ? '\n' : 0;
- else {
- /* end of header position confirmed */
- i_stream_skip(input, eoh);
- return;
- }
- }
-
- /* Leave enough space to go back "\r\nFrom" plus one for the
- end-of-headers check */
- startpos = i < 7 ? i : 7;
- i -= startpos;
-
- if (eoh != 0) {
- i_assert(i < eoh);
- eoh -= i;
- }
-
- i_stream_skip(input, i);
- }
-
- if (eoh != 0) {
- /* make sure we didn't end with \n\n or \n\r\n. In these
- cases the last [\r]\n doesn't belong to our message. */
- if (eoh < size && (msg[eoh] != '\r' || eoh < size-1)) {
- i_stream_skip(input, eoh);
- return;
- }
- }
-
- /* end of file, leave the last [\r]\n */
- msg = i_stream_get_data(input, &size);
- if (size == startpos && startpos > 0) {
- if (msg[startpos-1] == '\n')
- startpos--;
- if (startpos > 0 && msg[startpos-1] == '\r')
- startpos--;
- }
-
- i_stream_skip(input, startpos);
-}
-
-void mbox_skip_header(struct istream *input)
-{
- mbox_skip_forward(input, TRUE);
-}
-
-void mbox_skip_message(struct istream *input)
-{
- mbox_skip_forward(input, FALSE);
-}
-
-int mbox_verify_end_of_body(struct istream *input, uoff_t end_offset)
-{
- const unsigned char *data;
- size_t size;
-
- i_stream_seek(input, end_offset);
-
- /* read forward a bit */
- if (i_stream_read_data(input, &data, &size, 6) < 0)
- return FALSE;
-
- /* either there should be the next From-line,
- or [\r]\n at end of file */
- if (size > 0 && data[0] == '\r') {
- data++; size--;
- }
- if (size > 0) {
- if (data[0] != '\n')
- return FALSE;
-
- data++; size--;
- }
-
- return size == 0 ||
- (size >= 5 && strncmp((const char *) data, "From ", 5) == 0);
-}
-
-int mbox_mail_get_location(struct mail_index *index,
- struct mail_index_record *rec,
- uoff_t *offset, uoff_t *body_size)
-{
- struct message_size _body_size;
- const void *data;
- size_t size;
-
- if (offset != NULL) {
- if (!mail_cache_copy_fixed_field(index->cache, rec,
- MAIL_CACHE_LOCATION_OFFSET,
- offset, sizeof(*offset))) {
- mail_cache_set_corrupted(index->cache,
- "Missing location field for record %u",
- rec->uid);
- return FALSE;
- }
- }
-
- if (body_size != NULL) {
- if (mail_cache_copy_fixed_field(index->cache, rec,
- MAIL_CACHE_PHYSICAL_BODY_SIZE,
- body_size, sizeof(uoff_t)))
- return TRUE;
-
- if (!mail_cache_lookup_field(index->cache, rec,
- MAIL_CACHE_MESSAGEPART,
- &data, &size)) {
- mail_cache_set_corrupted(index->cache,
- "No cached body_size or message_part for "
- "record %u", rec->uid);
- return FALSE;
- }
- if (!message_part_deserialize_size(data, size,
- NULL, &_body_size)) {
- mail_cache_set_corrupted(index->cache,
- "Corrupted message_part for record %u",
- rec->uid);
- return FALSE;
- }
-
- if (body_size != NULL)
- *body_size = _body_size.physical_size;
- }
-
- return TRUE;
-}
-
-void mbox_read_headers(struct istream *input, buffer_t *dest)
-{
- struct message_header_parser_ctx *hdr_ctx;
- struct message_header_line *hdr;
-
- hdr_ctx = message_parse_header_init(input, NULL);
- while ((hdr = message_parse_header_next(hdr_ctx)) != NULL) {
- if (hdr->eoh) {
- buffer_append(dest, "\r\n", 2);
- break;
- }
-
- if ((*hdr->name == 'X' &&
- (strcasecmp(hdr->name, "X-UID") == 0 ||
- strcasecmp(hdr->name, "X-IMAPbase") == 0 ||
- strcasecmp(hdr->name, "X-Status") == 0 ||
- strcasecmp(hdr->name, "X-Keywords") == 0)) ||
- strcasecmp(hdr->name, "Content-Length") == 0 ||
- strcasecmp(hdr->name, "Status") == 0) {
- /* ignore */
- } else {
- if (!hdr->continued) {
- buffer_append(dest, hdr->name, hdr->name_len);
- buffer_append(dest, ": ", 2);
- }
- buffer_append(dest, hdr->value, hdr->value_len);
- buffer_append(dest, "\r\n", 2);
- }
- }
- message_parse_header_deinit(hdr_ctx);
-}
-
-struct mail_index *
-mbox_index_alloc(const char *mbox_path, const char *index_dir,
- const char *control_dir)
-{
- struct mail_index *index;
-
- i_assert(mbox_path != NULL);
-
- index = i_new(struct mail_index, 1);
- memcpy(index, &mbox_index, sizeof(struct mail_index));
-
- index->mbox_fd = -1;
- index->mbox_sync_counter = (unsigned int)-1;
- index->mailbox_readonly = access(mbox_path, W_OK) < 0;
-
- index->mailbox_path = i_strdup(mbox_path);
- index->control_dir = i_strdup(control_dir);
- mail_index_init(index, index_dir);
- return index;
-}
-
-static void mbox_index_free(struct mail_index *index)
-{
- mbox_file_close_fd(index);
- mail_index_close(index);
- i_free(index->dir);
- i_free(index->mailbox_path);
- i_free(index->control_dir);
- i_free(index);
-}
-
-static int mbox_index_set_lock(struct mail_index *index,
- enum mail_lock_type lock_type)
-{
- if (lock_type == MAIL_LOCK_UNLOCK)
- (void)mbox_unlock(index);
- return mail_index_set_lock(index, lock_type);
-}
-
-static int mbox_index_try_lock(struct mail_index *index,
- enum mail_lock_type lock_type)
-{
- if (lock_type == MAIL_LOCK_UNLOCK)
- (void)mbox_unlock(index);
- return mail_index_try_lock(index, lock_type);
-}
-
-static int mbox_index_expunge(struct mail_index *index,
- struct mail_index_record *first_rec,
- struct mail_index_record *last_rec,
- unsigned int first_seq, unsigned int last_seq,
- int external_change)
-{
- if (!mail_index_expunge(index, first_rec, last_rec,
- first_seq, last_seq, external_change))
- return FALSE;
-
- if (first_seq == 1) {
- /* Our message containing X-IMAPbase was deleted.
- Get it back there. */
- index->header->flags |= MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES |
- MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS;
- }
- return TRUE;
-}
-
-static int mbox_index_update_flags(struct mail_index *index,
- struct mail_index_record *rec,
- unsigned int seq,
- enum modify_type modify_type,
- enum mail_flags flags,
- int external_change)
-{
- enum mail_index_record_flag index_flags;
-
- if (!mail_index_update_flags(index, rec, seq,
- modify_type, flags, external_change))
- return FALSE;
-
- if (!external_change) {
- /* we'll just mark the message as dirty */
- index_flags = mail_cache_get_index_flags(index->cache, rec);
- if ((index_flags & MAIL_INDEX_FLAG_DIRTY) == 0) {
- if (mail_cache_lock(index->cache, FALSE) <= 0)
- return FALSE;
- mail_cache_unlock_later(index->cache);
-
- index_flags |= MAIL_INDEX_FLAG_DIRTY;
- mail_cache_update_index_flags(index->cache, rec,
- index_flags);
-
- index->header->flags |=
- MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES;
- }
- }
- return TRUE;
-}
-
-static struct mail_index_record *mbox_index_append(struct mail_index *index)
-{
- /* update last_uid in X-IMAPbase */
- index->header->flags |= MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES |
- MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS;
-
- return mail_index_append(index);
-}
-
-static time_t mbox_get_received_date(struct mail_index *index,
- struct mail_index_record *rec)
-{
- time_t date;
-
- if (mail_cache_copy_fixed_field(index->cache, rec,
- MAIL_CACHE_RECEIVED_DATE,
- &date, sizeof(date)))
- return date;
-
- mail_cache_set_corrupted(index->cache,
- "Missing internal date for record %u", rec->uid);
- return (time_t)-1;
-}
-
-struct mail_index mbox_index = {
- mail_index_open,
- mbox_index_free,
- mbox_index_set_lock,
- mbox_index_try_lock,
- mail_index_set_lock_notify_callback,
- mail_index_rebuild,
- mail_index_fsck,
- mbox_index_sync,
- mail_index_get_header,
- mail_index_lookup,
- mail_index_next,
- mail_index_lookup_uid_range,
- mbox_open_mail,
- mbox_get_received_date,
- mbox_index_expunge,
- mbox_index_update_flags,
- mbox_index_append,
- mail_index_get_last_error,
- mail_index_get_last_error_text,
-
- MAIL_INDEX_PRIVATE_FILL
-};
+++ /dev/null
-#ifndef __MBOX_INDEX_H
-#define __MBOX_INDEX_H
-
-#include "md5.h"
-#include "mail-index.h"
-
-/* Extra space to leave in X-Keywords header when rewriting mbox */
-#define MBOX_HEADER_EXTRA_SPACE 100
-
-struct mbox_header_context {
- struct mail_index *index;
- enum mail_flags flags;
- const char **custom_flags;
- struct md5_context md5;
- int received;
-
- unsigned int uid_validity, uid_last, uid;
-
- struct istream *input;
- uoff_t content_length;
-};
-
-int mbox_set_syscall_error(struct mail_index *index, const char *function);
-
-/* Make sure the mbox is opened. If reopen is TRUE, the file is closed first,
- which is useful when you want to be sure you're not accessing a deleted
- mbox file. */
-int mbox_file_open(struct mail_index *index);
-struct istream *mbox_get_stream(struct mail_index *index,
- enum mail_lock_type lock_type);
-void mbox_file_close_stream(struct mail_index *index);
-void mbox_file_close_fd(struct mail_index *index);
-
-void mbox_header_init_context(struct mbox_header_context *ctx,
- struct mail_index *index,
- struct istream *input);
-void mbox_header_cb(struct message_part *part,
- struct message_header_line *hdr, void *context);
-void mbox_keywords_parse(const unsigned char *value, size_t len,
- const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT],
- void (*func)(const unsigned char *, size_t,
- int, void *),
- void *context);
-int mbox_skip_crlf(struct istream *input);
-void mbox_skip_empty_lines(struct istream *input);
-void mbox_skip_header(struct istream *input);
-void mbox_skip_message(struct istream *input);
-int mbox_verify_end_of_body(struct istream *input, uoff_t end_offset);
-int mbox_mail_get_location(struct mail_index *index,
- struct mail_index_record *rec,
- uoff_t *offset, uoff_t *body_size);
-void mbox_read_headers(struct istream *input, buffer_t *dest);
-
-struct mail_index *
-mbox_index_alloc(const char *mbox_path, const char *index_dir,
- const char *control_dir);
-int mbox_index_sync(struct mail_index *index, int minimal_sync,
- enum mail_lock_type lock_type, int *changes);
-int mbox_sync_full(struct mail_index *index);
-struct istream *mbox_open_mail(struct mail_index *index,
- struct mail_index_record *rec,
- time_t *received_date, int *deleted);
-
-int mbox_index_append_stream(struct mail_index *index, struct istream *input);
-
-time_t mbox_from_parse_date(const unsigned char *msg, size_t size);
-const char *mbox_from_create(const char *sender, time_t time);
-
-int mbox_index_rewrite(struct mail_index *index);
-
-struct istream *i_stream_create_mbox(pool_t pool, struct istream *input,
- uoff_t offset, uoff_t body_size);
-
-#endif
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "mbox-index.h"
-#include "mbox-lock.h"
-#include "mail-index-util.h"
-
-#include <time.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-
-#ifdef HAVE_FLOCK
-# include <sys/file.h>
-#endif
-
-/* 0.1 .. 0.2msec */
-#define LOCK_RANDOM_USLEEP_TIME (100000 + (unsigned int)rand() % 100000)
-
-/* lock methods to use in wanted order */
-#define DEFAULT_LOCK_METHODS "dotlock fcntl"
-/* lock timeout */
-#define DEFAULT_LOCK_TIMEOUT 300
-/* assume stale dotlock if mbox file hasn't changed for n seconds */
-#define DEFAULT_DOTLOCK_CHANGE_TIMEOUT 30
-
-struct dotlock_context {
- struct mail_index *index;
- enum mail_lock_type lock_type;
- int last_stale;
-};
-
-static int lock_settings_initialized = FALSE;
-static int use_dotlock, use_fcntl_lock, use_flock, fcntl_before_flock;
-static int use_read_dotlock, lock_timeout, dotlock_change_timeout;
-
-static void mbox_init_lock_settings(void)
-{
- const char *str;
- const char *const *lock;
-
- use_dotlock = use_fcntl_lock = use_flock = fcntl_before_flock = FALSE;
-
- str = getenv("MBOX_LOCKS");
- if (str == NULL) str = DEFAULT_LOCK_METHODS;
- for (lock = t_strsplit_spaces(str, " "); *lock != NULL; lock++) {
- if (strcasecmp(*lock, "dotlock") == 0)
- use_dotlock = TRUE;
- else if (strcasecmp(*lock, "fcntl") == 0) {
- use_fcntl_lock = TRUE;
- fcntl_before_flock = use_flock == FALSE;
- } else if (strcasecmp(*lock, "flock") == 0)
- use_flock = TRUE;
- else
- i_fatal("MBOX_LOCKS: Invalid value %s", *lock);
- }
-
- use_read_dotlock = getenv("MBOX_READ_DOTLOCK") != NULL;
-
- str = getenv("MBOX_LOCK_TIMEOUT");
- lock_timeout = str == NULL ? DEFAULT_LOCK_TIMEOUT : atoi(str);
-
- str = getenv("MBOX_DOTLOCK_CHANGE_TIMEOUT");
- dotlock_change_timeout = str == NULL ?
- DEFAULT_DOTLOCK_CHANGE_TIMEOUT : atoi(str);
-
- lock_settings_initialized = TRUE;
-}
-
-#ifdef HAVE_FLOCK
-static int mbox_lock_flock(struct mail_index *index,
- enum mail_lock_type lock_type, time_t max_wait_time)
-{
- time_t now, last_notify;
-
- if (lock_type == MAIL_LOCK_EXCLUSIVE)
- lock_type = LOCK_EX;
- else if (lock_type == MAIL_LOCK_SHARED)
- lock_type = LOCK_SH;
- else
- lock_type = LOCK_UN;
-
- last_notify = 0;
- while (flock(index->mbox_fd, lock_type | LOCK_NB) < 0) {
- if (errno != EWOULDBLOCK) {
- mbox_set_syscall_error(index, "flock()");
- return FALSE;
- }
-
- if (max_wait_time == 0)
- return FALSE;
-
- now = time(NULL);
- if (now >= max_wait_time) {
- index->mailbox_lock_timeout = TRUE;
- index_set_error(index, "Timeout while waiting for "
- "release of flock() lock for mbox file "
- "%s", index->mailbox_path);
- return FALSE;
- }
-
- if (now != last_notify && index->lock_notify_cb != NULL) {
- last_notify = now;
- index->lock_notify_cb(MAIL_LOCK_NOTIFY_MAILBOX_ABORT,
- max_wait_time - now,
- index->lock_notify_context);
- }
-
- usleep(LOCK_RANDOM_USLEEP_TIME);
- }
-
- return TRUE;
-}
-#endif
-
-static int mbox_lock_fcntl(struct mail_index *index,
- enum mail_lock_type lock_type, time_t max_wait_time)
-{
- struct flock fl;
- time_t now;
- int wait_type;
-
- fl.l_type = MAIL_LOCK_TO_FLOCK(lock_type);
- fl.l_whence = SEEK_SET;
- fl.l_start = 0;
- fl.l_len = 0;
-
- wait_type = max_wait_time == 0 ? F_SETLK : F_SETLKW;
- while (fcntl(index->mbox_fd, wait_type, &fl) < 0) {
- if (errno != EINTR) {
- if (errno != EAGAIN && errno != EACCES)
- mbox_set_syscall_error(index, "fcntl()");
- return FALSE;
- }
-
- now = time(NULL);
- if (max_wait_time != 0 && now >= max_wait_time) {
- index->mailbox_lock_timeout = TRUE;
- index_set_error(index, "Timeout while waiting for "
- "release of fcntl() lock for mbox file "
- "%s", index->mailbox_path);
- return FALSE;
- }
-
- if (index->lock_notify_cb != NULL) {
- index->lock_notify_cb(MAIL_LOCK_NOTIFY_MAILBOX_ABORT,
- max_wait_time - now,
- index->lock_notify_context);
- }
- }
- return TRUE;
-}
-
-static int mbox_file_locks(struct mail_index *index,
- enum mail_lock_type lock_type, time_t max_wait_time)
-{
- struct stat st;
-
- /* now we need to have the file itself locked. open it if needed. */
- if (stat(index->mailbox_path, &st) < 0)
- return mbox_set_syscall_error(index, "stat()");
-
- if (st.st_dev != index->mbox_dev || st.st_ino != index->mbox_ino)
- mbox_file_close_fd(index);
-
- if (index->mbox_fd == -1) {
- if (!mbox_file_open(index)) {
- (void)mbox_unlock(index);
- return FALSE;
- }
- }
-
- if (use_fcntl_lock && fcntl_before_flock) {
- if (!mbox_lock_fcntl(index, lock_type, max_wait_time))
- return FALSE;
- }
-#ifdef HAVE_FLOCK
- if (use_flock) {
- if (!mbox_lock_flock(index, lock_type, max_wait_time))
- return FALSE;
- }
-#endif
- if (use_fcntl_lock && !fcntl_before_flock) {
- if (!mbox_lock_fcntl(index, lock_type, max_wait_time))
- return FALSE;
- }
- return TRUE;
-}
-
-static int mbox_file_unlock(struct mail_index *index)
-{
- int failed = FALSE;
-
-#ifdef HAVE_FLOCK
- if (use_flock && !mbox_lock_flock(index, MAIL_LOCK_UNLOCK, 0))
- failed = TRUE;
-#endif
- if (use_fcntl_lock &&
- !mbox_lock_fcntl(index, MAIL_LOCK_UNLOCK, 0))
- failed = TRUE;
-
- return !failed;
-}
-
-static int dotlock_callback(unsigned int secs_left, int stale, void *context)
-{
- struct dotlock_context *ctx = context;
-
- if (stale && !ctx->last_stale) {
- if (!mbox_file_locks(ctx->index, ctx->lock_type, 0)) {
- /* we couldn't get fcntl/flock - it's really locked */
- ctx->last_stale = TRUE;
- return FALSE;
- }
- (void)mbox_file_unlock(ctx->index);
- }
- ctx->last_stale = stale;
-
- if (ctx->index->lock_notify_cb != NULL) {
- ctx->index->lock_notify_cb(stale ?
- MAIL_LOCK_NOTIFY_MAILBOX_OVERRIDE :
- MAIL_LOCK_NOTIFY_MAILBOX_ABORT,
- secs_left,
- ctx->index->lock_notify_context);
- }
- return TRUE;
-}
-
-int mbox_lock(struct mail_index *index, enum mail_lock_type lock_type)
-{
- time_t max_wait_time;
- int ret;
-
- /* index must be locked before mbox file, to avoid deadlocks */
- i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
-
- /* allow only unlock -> shared/exclusive or exclusive -> shared */
- i_assert(lock_type == MAIL_LOCK_SHARED ||
- lock_type == MAIL_LOCK_EXCLUSIVE);
- i_assert(lock_type != MAIL_LOCK_EXCLUSIVE ||
- index->mbox_lock_type != MAIL_LOCK_SHARED);
-
- if (index->mbox_lock_type == lock_type)
- return TRUE;
-
- if (!lock_settings_initialized)
- mbox_init_lock_settings();
-
- max_wait_time = time(NULL) + lock_timeout;
-
- /* make .lock file first to protect overwriting the file */
- if (use_dotlock && index->mbox_dotlock.ino == 0) {
- struct dotlock_context ctx;
-
- ctx.index = index;
- ctx.lock_type = lock_type;
- ctx.last_stale = -1;
-
- ret = file_lock_dotlock(index->mailbox_path, NULL,
- lock_type == MAIL_LOCK_SHARED &&
- !use_read_dotlock, lock_timeout,
- dotlock_change_timeout, 0,
- dotlock_callback, &ctx,
- &index->mbox_dotlock);
-
- if (ret < 0) {
- mbox_set_syscall_error(index, "file_lock_dotlock()");
- return FALSE;
- }
- if (ret == 0) {
- index_set_error(index, "Timeout while waiting for "
- "release of dotlock for mbox %s",
- index->mailbox_path);
- index->mailbox_lock_timeout = TRUE;
- return FALSE;
- }
- }
-
- index->mbox_lock_type = lock_type;
- if (!mbox_file_locks(index, index->mbox_lock_type, max_wait_time)) {
- (void)mbox_unlock(index);
- return FALSE;
- }
-
- return TRUE;
-}
-
-int mbox_unlock(struct mail_index *index)
-{
- int failed;
-
- index->mbox_lock_counter++;
-
- if (index->mbox_lock_type == MAIL_LOCK_UNLOCK)
- return TRUE;
-
- failed = FALSE;
- if (index->mbox_fd != -1) {
- if (!mbox_file_unlock(index))
- failed = TRUE;
- }
-
- if (index->mbox_dotlock.ino != 0) {
- if (file_unlock_dotlock(index->mailbox_path,
- &index->mbox_dotlock) <= 0) {
- mbox_set_syscall_error(index, "file_unlock_dotlock()");
- failed = TRUE;
- }
- index->mbox_dotlock.ino = 0;
- }
-
- /* make sure we don't keep mmap() between locks - there could have
- been changes to file size which would break things. or actually
- it'd break only if file was shrinked+grown back to exact size,
- but still possible :) */
- mbox_file_close_stream(index);
-
- index->mbox_lock_type = MAIL_LOCK_UNLOCK;
- return !failed;
-}
+++ /dev/null
-#ifndef __MBOX_LOCK_H
-#define __MBOX_LOCK_H
-
-/* NOTE: if mbox file is not open, it's opened. if it is open but file has
- been overwritten (ie. inode has changed), it's reopened. */
-int mbox_lock(struct mail_index *index, enum mail_lock_type lock_type);
-int mbox_unlock(struct mail_index *index);
-
-#endif
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "istream.h"
-#include "mbox-index.h"
-#include "mail-index-util.h"
-
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-
-struct istream *mbox_open_mail(struct mail_index *index,
- struct mail_index_record *rec,
- time_t *received_date, int *deleted)
-{
- struct istream *input;
- uoff_t offset, body_size;
-
- i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
-
- *deleted = FALSE;
-
- /* check for inconsistency here, to avoid extra error messages */
- if (index->inconsistent)
- return NULL;
-
- if (!mbox_mail_get_location(index, rec, &offset, &body_size))
- return NULL;
-
- input = mbox_get_stream(index, MAIL_LOCK_SHARED);
- if (input == NULL)
- return NULL;
-
- if (received_date != NULL)
- *received_date = index->get_received_date(index, rec);
-
- i_assert(index->mbox_sync_counter == index->mbox_lock_counter);
- return i_stream_create_mbox(default_pool, input, offset, body_size);
-}
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "ioloop.h"
-#include "istream.h"
-#include "ostream.h"
-#include "file-set-size.h"
-#include "str.h"
-#include "write-full.h"
-#include "mbox-index.h"
-#include "mbox-lock.h"
-#include "mail-index-util.h"
-#include "mail-custom-flags.h"
-#include "mail-cache.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-
-struct mbox_rewrite_context {
- struct ostream *output;
-
- uoff_t content_length;
- unsigned int seq, uid;
- unsigned int msg_flags;
- const char **custom_flags;
-
- unsigned int uid_validity;
- unsigned int uid_last;
- char *x_keywords;
-
- unsigned int ximapbase_found:1;
- unsigned int xuid_found:1;
- unsigned int status_found:1;
- unsigned int xstatus_found:1;
- unsigned int content_length_found:1;
-};
-
-/* Remove dirty flag from all messages */
-static int reset_dirty_flags(struct mail_index *index)
-{
- struct mail_index_record *rec;
- enum mail_index_record_flag index_flags;
-
- if (mail_cache_lock(index->cache, FALSE) <= 0)
- return FALSE;
- mail_cache_unlock_later(index->cache);
-
- rec = index->lookup(index, 1);
- while (rec != NULL) {
- index_flags = mail_cache_get_index_flags(index->cache, rec);
- if ((index_flags & MAIL_INDEX_FLAG_DIRTY) != 0) {
- index_flags &= ~MAIL_INDEX_FLAG_DIRTY;
- if (!mail_cache_update_index_flags(index->cache,
- rec, index_flags))
- return FALSE;
- }
-
- rec = index->next(index, rec);
- }
-
- index->header->flags &= ~(MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES |
- MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS);
- return TRUE;
-}
-
-static int mbox_write(struct mail_index *index, struct istream *input,
- struct ostream *output, uoff_t end_offset)
-{
- int failed;
-
- i_assert(input->v_offset <= end_offset);
-
- input = i_stream_create_limit(default_pool, input, 0, end_offset);
- if (o_stream_send_istream(output, input) < 0) {
- index_set_error(index, "Error rewriting mbox file %s: %s",
- index->mailbox_path,
- strerror(output->stream_errno));
- failed = TRUE;
- } else if (input->v_offset < end_offset) {
- /* sync should have noticed it.. */
- index_set_error(index, "Error rewriting mbox file %s: "
- "Unexpected end of file", index->mailbox_path);
- failed = TRUE;
- } else {
- failed = FALSE;
- }
-
- i_stream_unref(input);
- return !failed;
-}
-
-static int mbox_write_ximapbase(struct mbox_rewrite_context *ctx)
-{
- const char *str;
- int i;
-
- str = t_strdup_printf("X-IMAPbase: %u %u",
- ctx->uid_validity, ctx->uid_last);
- if (o_stream_send_str(ctx->output, str) < 0)
- return FALSE;
-
- for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
- if (ctx->custom_flags[i] != NULL) {
- if (o_stream_send(ctx->output, " ", 1) < 0)
- return FALSE;
-
- if (o_stream_send_str(ctx->output,
- ctx->custom_flags[i]) < 0)
- return FALSE;
- }
- }
-
- if (o_stream_send(ctx->output, "\n", 1) < 0)
- return FALSE;
-
- return TRUE;
-}
-
-static int mbox_write_xuid(struct mbox_rewrite_context *ctx)
-{
- const char *str;
-
- str = t_strdup_printf("X-UID: %u\n", ctx->uid);
-
- if (o_stream_send_str(ctx->output, str) < 0)
- return FALSE;
-
- return TRUE;
-}
-
-static int mbox_write_xkeywords(struct mbox_rewrite_context *ctx,
- const char *x_keywords, uoff_t wanted_offset,
- int force_filler)
-{
- unsigned int field;
- int i;
-
- if ((ctx->msg_flags & MAIL_CUSTOM_FLAGS_MASK) == 0 &&
- x_keywords == NULL && !force_filler &&
- ctx->output->offset + sizeof("X-Keywords:")+1 >= wanted_offset) {
- /* nothing to do, and not enough extra space to write the
- filler. Do it only if there's space for "X-Keywords: \n" */
- return TRUE;
- }
-
- if (o_stream_send_str(ctx->output, "X-Keywords:") < 0)
- return FALSE;
-
- field = 1 << MAIL_CUSTOM_FLAG_1_BIT;
- for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++, field <<= 1) {
- if ((ctx->msg_flags & field) && ctx->custom_flags[i] != NULL) {
- if (o_stream_send(ctx->output, " ", 1) < 0)
- return FALSE;
-
- if (o_stream_send_str(ctx->output,
- ctx->custom_flags[i]) < 0)
- return FALSE;
- }
- }
-
- if (x_keywords != NULL) {
- /* X-Keywords that aren't custom flags */
- if (o_stream_send(ctx->output, " ", 1) < 0)
- return FALSE;
-
- if (o_stream_send_str(ctx->output, x_keywords) < 0)
- return FALSE;
- }
-
- /* fill the rest with spaces. -1 for \n */
- if (ctx->output->offset < wanted_offset-1 || force_filler) {
- char buf[1024];
- uoff_t fill_left;
-
- fill_left = force_filler ? MBOX_HEADER_EXTRA_SPACE :
- wanted_offset-1 - ctx->output->offset;
- memset(buf, ' ', sizeof(buf));
- while (fill_left > sizeof(buf)) {
- if (o_stream_send(ctx->output, buf, sizeof(buf)) < 0)
- return FALSE;
- fill_left -= sizeof(buf);
- }
- if (o_stream_send(ctx->output, buf, fill_left) < 0)
- return FALSE;
- }
-
- if (o_stream_send(ctx->output, "\n", 1) < 0)
- return FALSE;
-
- return TRUE;
-}
-
-static int mbox_write_status(struct mbox_rewrite_context *ctx,
- const char *status)
-{
- const char *str;
-
- str = (ctx->msg_flags & MAIL_SEEN) ? "Status: RO" : "Status: O";
- if (status != NULL)
- str = t_strconcat(str, status, NULL);
-
- if (o_stream_send_str(ctx->output, str) < 0)
- return FALSE;
- if (o_stream_send(ctx->output, "\n", 1) < 0)
- return FALSE;
-
- return TRUE;
-}
-
-static int mbox_write_xstatus(struct mbox_rewrite_context *ctx,
- const char *x_status)
-{
- const char *str;
-
- /* X-Status field */
- if ((ctx->msg_flags & (MAIL_SYSTEM_FLAGS_MASK^MAIL_SEEN)) == 0 &&
- x_status == NULL)
- return TRUE;
-
- str = t_strconcat("X-Status: ",
- (ctx->msg_flags & MAIL_ANSWERED) ? "A" : "",
- (ctx->msg_flags & MAIL_DELETED) ? "D" : "",
- (ctx->msg_flags & MAIL_FLAGGED) ? "F" : "",
- (ctx->msg_flags & MAIL_DRAFT) ? "T" : "",
- x_status, NULL);
-
- if (o_stream_send_str(ctx->output, str) < 0)
- return FALSE;
- if (o_stream_send(ctx->output, "\n", 1) < 0)
- return FALSE;
-
- return TRUE;
-}
-
-static int mbox_write_content_length(struct mbox_rewrite_context *ctx)
-{
- char str[MAX_INT_STRLEN+30];
-
- i_snprintf(str, sizeof(str), "Content-Length: %"PRIuUOFF_T"\n",
- ctx->content_length);
-
- if (o_stream_send_str(ctx->output, str) < 0)
- return FALSE;
- return TRUE;
-}
-
-static const char *strip_chars(const unsigned char *value, size_t value_len,
- const char *list)
-{
- /* @UNSAFE: leave only unknown flags, very likely none */
- char *ret, *p;
- size_t i;
-
- ret = p = t_buffer_get(value_len+1);
- for (i = 0; i < value_len; i++) {
- if (strchr(list, value[i]) == NULL)
- *p++ = value[i];
- }
-
- if (ret == p)
- return NULL;
- *p = '\0';
- t_buffer_alloc((size_t) (p-ret)+1);
- return ret;
-}
-
-static void update_stripped_custom_flags(const unsigned char *value, size_t len,
- int index, void *context)
-{
- string_t *str = context;
-
- if (index < 0) {
- /* not found, keep it */
- if (str_len(str) != 0)
- str_append_c(str, ' ');
- str_append_n(str, value, len);
- }
-}
-
-static const char *strip_custom_flags(const unsigned char *value, size_t len,
- struct mbox_rewrite_context *ctx)
-{
- string_t *str;
-
- str = t_str_new(len+1);
- mbox_keywords_parse(value, len, ctx->custom_flags,
- update_stripped_custom_flags, str);
- return str_len(str) == 0 ? NULL : str_c(str);
-}
-
-static int write_header(struct mbox_rewrite_context *ctx,
- struct message_header_line *hdr)
-{
- const char *str;
-
- switch (hdr->name_len) {
- case 5:
- if (strcasecmp(hdr->name, "X-UID") == 0) {
- if (ctx->xuid_found)
- return TRUE;
-
- ctx->xuid_found = TRUE;
- return mbox_write_xuid(ctx);
- }
- break;
- case 6:
- if (strcasecmp(hdr->name, "Status") == 0) {
- if (ctx->status_found)
- return TRUE;
- if (hdr->continues) {
- hdr->use_full_value = TRUE;
- return TRUE;
- }
-
- ctx->status_found = TRUE;
- str = strip_chars(hdr->full_value,
- hdr->full_value_len, "RO");
- return mbox_write_status(ctx, str);
- }
- break;
- case 8:
- if (strcasecmp(hdr->name, "X-Status") == 0) {
- if (ctx->xstatus_found)
- return TRUE;
- if (hdr->continues) {
- hdr->use_full_value = TRUE;
- return TRUE;
- }
-
- ctx->xstatus_found = TRUE;
- str = strip_chars(hdr->full_value,
- hdr->full_value_len, "ADFT");
- return mbox_write_xstatus(ctx, str);
- }
- break;
- case 10:
- if (strcasecmp(hdr->name, "X-Keywords") == 0) {
- if (ctx->x_keywords != NULL)
- return TRUE;
- if (hdr->continues) {
- hdr->use_full_value = TRUE;
- return TRUE;
- }
-
- str = strip_custom_flags(hdr->full_value,
- hdr->full_value_len, ctx);
- ctx->x_keywords = i_strdup(str);
- return TRUE;
- } else if (strcasecmp(hdr->name, "X-IMAPbase") == 0) {
- if (ctx->seq != 1 || ctx->ximapbase_found)
- return TRUE;
-
- ctx->ximapbase_found = TRUE;
- return mbox_write_ximapbase(ctx);
- }
- break;
- case 14:
- if (strcasecmp(hdr->name, "Content-Length") == 0) {
- if (ctx->content_length_found)
- return TRUE;
-
- ctx->content_length_found = TRUE;
- return mbox_write_content_length(ctx);
- }
- break;
- }
-
- if (!hdr->eoh) {
- /* save this header */
- if (!hdr->continued) {
- (void)o_stream_send(ctx->output, hdr->name,
- hdr->name_len);
- (void)o_stream_send(ctx->output, ": ", 2);
- }
- (void)o_stream_send(ctx->output, hdr->value, hdr->value_len);
- if (!hdr->no_newline)
- (void)o_stream_send(ctx->output, "\n", 1);
- }
-
- return !ctx->output->closed;
-}
-
-static int mbox_write_header(struct mail_index *index,
- struct mail_index_record *rec, unsigned int seq,
- struct istream *input, struct ostream *output,
- uoff_t dirty_offset,
- uoff_t *hdr_input_size, uoff_t body_size)
-{
- /* We need to update fields that define message flags. Standard fields
- are stored in Status and X-Status. For custom flags we use
- uw-imapd compatible format, by first listing them in first message's
- X-IMAPbase field and actually defining them in X-Keywords field.
-
- Format of X-IMAPbase is: <UID validity> <last used UID> <flag names>
-
- We don't want to sync our UIDs with the mbox file, so the UID
- validity is always kept different from our internal UID validity.
- Last used UID is also not updated, and set to 0 initially.
- */
- struct mbox_rewrite_context ctx;
- struct message_header_parser_ctx *hdr_ctx;
- struct message_header_line *hdr;
- struct message_size hdr_size;
- struct istream *hdr_input;
- uoff_t offset;
- int force_filler;
-
- t_push();
-
- /* parse the header, write the fields we don't want to change */
- memset(&ctx, 0, sizeof(ctx));
- ctx.output = output;
- ctx.content_length = body_size;
- ctx.seq = seq;
- ctx.uid = rec->uid;
- ctx.msg_flags = rec->msg_flags;
- ctx.uid_validity = index->header->uid_validity;
- ctx.uid_last = index->header->next_uid-1;
- ctx.custom_flags = mail_custom_flags_list_get(index->custom_flags);
-
- if (body_size != 0) {
- hdr_input = input;
- i_stream_ref(hdr_input);
- } else {
- /* possibly broken message, find the next From-line
- and make sure header parser won't pass it. */
- offset = input->v_offset;
- mbox_skip_header(input);
- i_stream_seek(input, offset);
- hdr_input = i_stream_create_limit(default_pool, input, 0,
- input->v_offset);
- }
-
- hdr_ctx = message_parse_header_init(hdr_input, &hdr_size);
- while ((hdr = message_parse_header_next(hdr_ctx)) != NULL) {
- t_push();
- write_header(&ctx, hdr);
- t_pop();
- }
- message_parse_header_deinit(hdr_ctx);
- *hdr_input_size = hdr_size.physical_size;
-
- i_stream_unref(hdr_input);
-
- /* append the flag fields */
- if (seq == 1 && !ctx.ximapbase_found) {
- /* write X-IMAPbase header to first message */
- (void)mbox_write_ximapbase(&ctx);
- }
-
- force_filler = !ctx.xuid_found;
- if (!ctx.status_found)
- (void)mbox_write_status(&ctx, NULL);
- if (!ctx.xstatus_found)
- (void)mbox_write_xstatus(&ctx, NULL);
- if (!ctx.xuid_found)
- (void)mbox_write_xuid(&ctx);
- if (!ctx.content_length_found)
- (void)mbox_write_content_length(&ctx);
-
- /* write the x-keywords header last so it can fill the extra space
- with spaces. -1 is for ending \n. */
- (void)mbox_write_xkeywords(&ctx, ctx.x_keywords,
- input->v_offset - dirty_offset - 1,
- force_filler);
- i_free(ctx.x_keywords);
-
- t_pop();
-
- /* empty line ends headers */
- (void)o_stream_send(output, "\n", 1);
-
- return TRUE;
-}
-
-static int fd_copy(struct mail_index *index, int in_fd, int out_fd,
- uoff_t out_offset, uoff_t size)
-{
- struct istream *input, *input2;
- struct ostream *output;
- struct stat st;
- int ret;
-
- i_assert(out_offset <= OFF_T_MAX);
-
- /* first grow the file to wanted size, to make sure we don't run out
- of disk space */
- if (fstat(out_fd, &st) < 0) {
- mbox_set_syscall_error(index, "fstat()");
- return -1;
- }
-
- if ((uoff_t)st.st_size < out_offset + size) {
- if (file_set_size(out_fd, (off_t)(out_offset + size)) < 0) {
- mbox_set_syscall_error(index, "file_set_size()");
- (void)ftruncate(out_fd, st.st_size);
- return -1;
- }
- }
-
- if (lseek(out_fd, (off_t)out_offset, SEEK_SET) < 0) {
- mbox_set_syscall_error(index, "lseek()");
- (void)ftruncate(out_fd, st.st_size);
- return -1;
- }
-
- t_push();
-
- input = i_stream_create_file(in_fd, pool_datastack_create(),
- 1024*256, FALSE);
- input2 = i_stream_create_limit(pool_datastack_create(), input, 0, size);
-
- output = o_stream_create_file(out_fd, pool_datastack_create(),
- 1024, FALSE);
- o_stream_set_blocking(output, 60000, NULL, NULL);
-
- ret = o_stream_send_istream(output, input2);
- if (ret < 0) {
- errno = output->stream_errno;
- mbox_set_syscall_error(index, "o_stream_send_istream()");
- }
-
- o_stream_unref(output);
- i_stream_unref(input2);
- i_stream_unref(input);
- t_pop();
-
- return ret;
-}
-
-static int dirty_flush(struct mail_index *index, uoff_t dirty_offset,
- struct ostream *output, int output_fd)
-{
- if (output->offset == 0)
- return TRUE;
-
- if (o_stream_flush(output) < 0) {
- mbox_set_syscall_error(index, "o_stream_flush()");
- return FALSE;
- }
-
- /* POSSIBLE DATA LOSS HERE. We're writing to the mbox file,
- so if we get killed here before finished, we'll lose some
- bytes. I can't really think of any way to fix this,
- rename() is problematic too especially because of file
- locking issues (new mail could be lost).
-
- Usually we're moving the data by just a few bytes, so
- the data loss should never be more than those few bytes..
- If we moved more, we could have written the file from end
- to beginning in blocks (it'd be a bit slow to do it in
- blocks of ~1-10 bytes which is the usual case, so we don't
- bother).
-
- Also, we might as well be shrinking the file, in which
- case we can't lose data. */
- if (fd_copy(index, output_fd, index->mbox_fd,
- dirty_offset, output->offset) < 0)
- return FALSE;
-
- /* All ok. Just make sure the timestamps of index and
- mbox differ, so index will be updated at next sync */
- index->sync_stamp = 0;
-
- if (o_stream_seek(output, 0) < 0) {
- mbox_set_syscall_error(index, "o_stream_seek()");
- return FALSE;
- }
- return TRUE;
-}
-
-#define INDEX_DIRTY_FLAGS \
- (MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES | \
- MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS)
-
-int mbox_index_rewrite(struct mail_index *index)
-{
- /* Write messages beginning from the first dirty one to temp file,
- then copy it over the mbox file. This may create data loss if
- interrupted (see below). This rewriting relies quite a lot on
- valid header/body sizes which fsck() should have ensured. */
- struct mail_index_record *rec;
- struct istream *input;
- struct ostream *output;
- uoff_t offset, hdr_size, body_size, dirty_offset;
- const char *path;
- unsigned int seq;
- int tmp_fd, failed, dirty, dirty_found, rewrite, no_locking;
-
- i_assert(!index->mailbox_readonly);
- i_assert(index->lock_type == MAIL_LOCK_UNLOCK ||
- (index->lock_type == MAIL_LOCK_EXCLUSIVE &&
- index->mbox_lock_type == MAIL_LOCK_EXCLUSIVE));
-
- no_locking = index->mbox_lock_type == MAIL_LOCK_EXCLUSIVE;
- if (!no_locking) {
- if (!index->set_lock(index, MAIL_LOCK_SHARED))
- return FALSE;
- }
-
- rewrite = (index->header->flags & INDEX_DIRTY_FLAGS) &&
- index->header->messages_count > 0;
-
- if (!no_locking) {
- if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
- return FALSE;
- }
-
- if (!rewrite) {
- /* no need to rewrite */
- return TRUE;
- }
-
- /* kludgy .. but we need to force resyncing */
- index->mbox_rewritten = TRUE;
-
- tmp_fd = -1; input = NULL;
- failed = TRUE; rewrite = FALSE;
- do {
- if (!no_locking) {
- if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
- break;
-
- if (!index->sync_and_lock(index, FALSE,
- MAIL_LOCK_EXCLUSIVE, NULL))
- break;
- }
-
- input = mbox_get_stream(index, MAIL_LOCK_EXCLUSIVE);
- if (input == NULL)
- break;
-
- if ((index->header->flags & INDEX_DIRTY_FLAGS) == 0) {
- /* fsck() figured out there's no dirty messages
- after all */
- failed = FALSE; rewrite = FALSE;
- break;
- }
-
- tmp_fd = mail_index_create_temp_file(index, &path);
- if (tmp_fd == -1)
- break;
-
- failed = FALSE; rewrite = TRUE;
- } while (0);
-
- if (!rewrite) {
- if (!no_locking) {
- if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
- failed = TRUE;
- }
- if (input != NULL)
- i_stream_unref(input);
- return !failed;
- }
-
- if (index->header->flags & MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS) {
- /* need to update X-IMAPbase in first message */
- dirty_found = TRUE;
- } else {
- dirty_found = FALSE;
- }
- dirty_offset = 0;
-
- /* note: we can't use data_stack_pool with output stream because it's
- being written to inside t_push() .. t_pop() calls */
- output = o_stream_create_file(tmp_fd, system_pool, 8192, FALSE);
- o_stream_set_blocking(output, 60000, NULL, NULL);
-
- failed = FALSE; seq = 1;
- rec = index->lookup(index, 1);
- while (rec != NULL) {
- if (dirty_found)
- dirty = FALSE;
- else {
- dirty = (mail_cache_get_index_flags(index->cache, rec) &
- MAIL_INDEX_FLAG_DIRTY) != 0;
- }
-
- if (dirty_found || dirty) {
- /* get offset to beginning of mail headers */
- if (!mbox_mail_get_location(index, rec, &offset,
- &body_size)) {
- /* fsck should have fixed it */
- failed = TRUE;
- break;
- }
-
- if (offset < input->v_offset) {
- mail_cache_set_corrupted(index->cache,
- "Invalid message offset");
- failed = TRUE;
- break;
- }
-
- if (!dirty_found) {
- /* first dirty message */
- dirty_found = TRUE;
- dirty_offset = offset;
-
- i_stream_seek(input, dirty_offset);
- }
-
- /* write the From-line */
- if (!mbox_write(index, input, output, offset)) {
- failed = TRUE;
- break;
- }
-
- /* write header, updating flag fields */
- if (!mbox_write_header(index, rec, seq, input, output,
- dirty_offset,
- &hdr_size, body_size)) {
- failed = TRUE;
- break;
- }
- offset += hdr_size;
- i_assert(input->v_offset == offset);
-
- if (dirty_found &&
- offset - dirty_offset == output->offset) {
- /* no need to write more, flush */
- if (!dirty_flush(index, dirty_offset,
- output, tmp_fd)) {
- failed = TRUE;
- break;
- }
- dirty_found = FALSE;
- } else {
- /* write body */
- offset += body_size;
- if (!mbox_write(index, input, output, offset)) {
- failed = TRUE;
- break;
- }
- }
- }
-
- seq++;
- rec = index->next(index, rec);
- }
-
- if (!failed && dirty_found) {
- /* end with \n */
- (void)o_stream_send(output, "\n", 1);
- }
-
- if (output->closed) {
- errno = output->stream_errno;
- mbox_set_syscall_error(index, "write()");
- failed = TRUE;
- }
-
- if (!failed && dirty_found) {
- uoff_t dirty_size = output->offset;
-
- if (!dirty_flush(index, dirty_offset, output, tmp_fd))
- failed = TRUE;
- else {
- /* we may have shrinked the file */
- i_assert(dirty_offset + dirty_size <= OFF_T_MAX);
- if (ftruncate(index->mbox_fd,
- (off_t)(dirty_offset + dirty_size)) < 0) {
- mbox_set_syscall_error(index, "ftruncate()");
- failed = TRUE;
- }
- }
- }
-
- if (!failed) {
- if (!reset_dirty_flags(index))
- failed = TRUE;
- }
-
- i_stream_unref(input);
- o_stream_unref(output);
-
- if (!no_locking) {
- if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
- failed = TRUE;
- }
-
- (void)unlink(path);
-
- if (close(tmp_fd) < 0)
- index_file_set_syscall_error(index, path, "close()");
- return !failed;
-}
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "istream.h"
-#include "hex-binary.h"
-#include "message-parser.h"
-#include "message-part-serialize.h"
-#include "mbox-index.h"
-#include "mbox-lock.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-
-static void skip_line(struct istream *input)
-{
- const unsigned char *msg;
- size_t i, size;
- int ret;
-
- while ((ret = i_stream_read_data(input, &msg, &size, 0)) > 0) {
- for (i = 0; i < size; i++) {
- if (msg[i] == '\n') {
- i_stream_skip(input, i+1);
- return;
- }
- }
-
- i_stream_skip(input, i);
- }
-}
-
-static int verify_header(struct mail_index *index,
- struct mail_index_record *rec,
- unsigned int uid, unsigned char current_digest[16])
-{
- const void *old_digest;
- size_t size;
-
- if (uid != 0) {
- /* X-UID header - no need to check more */
- return uid == rec->uid;
- }
-
- /* check if MD5 sums match */
- if (!mail_cache_lookup_field(index->cache, rec, MAIL_CACHE_MD5,
- &old_digest, &size))
- return FALSE;
-
- return memcmp(old_digest, current_digest, 16) == 0;
-}
-
-static int mbox_check_uidvalidity(struct mail_index *index,
- unsigned int uid_validity)
-{
- if (uid_validity == index->header->uid_validity)
- return TRUE;
-
- index->header->flags |= MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES |
- MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS;
-
- if (uid_validity == 0) {
- /* X-IMAPbase header isn't written yet */
- } else {
- /* UID validity has changed - rebuild whole index */
- index->set_flags |= MAIL_INDEX_HDR_FLAG_REBUILD;
- index->inconsistent = TRUE;
- return FALSE;
- }
-
- return TRUE;
-}
-
-static int match_next_record(struct mail_index *index,
- struct mail_index_record *rec,
- unsigned int *seq, struct istream *input,
- struct mail_index_record **next_rec, int *dirty)
-{
- struct mbox_header_context ctx;
- struct mail_index_record *first_rec, *last_rec;
- struct istream *hdr_input;
- enum mail_index_record_flag index_flags;
- uoff_t header_offset, body_offset, offset, body_size, eoh_offset;
- unsigned char current_digest[16];
- unsigned int first_seq, last_seq;
- int ret, hdr_parsed;
-
- *next_rec = NULL;
-
- /* skip the From-line */
- skip_line(input);
- header_offset = input->v_offset;
-
- first_rec = last_rec = NULL;
- first_seq = last_seq = 0;
- ret = 0; body_offset = 0; eoh_offset = (uoff_t)-1; hdr_parsed = FALSE;
- do {
- if (!mbox_mail_get_location(index, rec, &offset, &body_size))
- return -1;
-
- if (body_size == 0 && eoh_offset == (uoff_t)-1) {
- /* possibly broken message, find the next From-line
- and make sure header parser won't pass it. */
- i_stream_seek(input, header_offset);
- mbox_skip_header(input);
- eoh_offset = input->v_offset;
- hdr_parsed = FALSE;
- }
-
- if (!hdr_parsed) {
- /* get the MD5 sum of fixed headers and the current
- message flags in Status and X-Status fields */
- if (eoh_offset == (uoff_t)-1)
- hdr_input = input;
- else {
- hdr_input = i_stream_create_limit(default_pool,
- input, 0, eoh_offset);
- }
- i_stream_seek(hdr_input, header_offset);
-
- mbox_header_init_context(&ctx, index, hdr_input);
- message_parse_header(NULL, hdr_input, NULL,
- mbox_header_cb, &ctx);
-
- hdr_parsed = TRUE;
- body_offset = hdr_input->v_offset;
-
- if (eoh_offset != (uoff_t)-1)
- i_stream_unref(hdr_input);
- hdr_input = NULL;
- md5_final(&ctx.md5, current_digest);
-
- if (*seq == 1) {
- if (!mbox_check_uidvalidity(index,
- ctx.uid_validity)) {
- /* uidvalidity changed, abort */
- return -1;
- }
-
- if (ctx.uid_last >= index->header->next_uid) {
- /* last_uid larger than ours */
- index->header->next_uid =
- ctx.uid_last+1;
- }
- }
- }
-
- if (verify_header(index, rec, ctx.uid, current_digest) &&
- mbox_verify_end_of_body(input, body_offset + body_size)) {
- /* valid message */
-
- /* update flags, unless we've changed them */
- index_flags =
- mail_cache_get_index_flags(index->cache, rec);
- if ((index_flags & MAIL_INDEX_FLAG_DIRTY) == 0) {
- if (!index->update_flags(index, rec, *seq,
- MODIFY_REPLACE,
- ctx.flags, TRUE))
- return -1;
- } else if (rec->msg_flags == ctx.flags) {
- /* flags are same, it's not dirty anymore */
- index_flags &= ~MAIL_INDEX_FLAG_DIRTY;
- mail_cache_update_index_flags(index->cache,
- rec, index_flags);
- } else {
- *dirty = TRUE;
- }
-
- /* update location */
- if (offset != header_offset) {
- if (!mail_cache_update_location_offset(
- index->cache, rec, header_offset))
- return -1;
- }
- ret = 1;
- break;
- }
-
- /* try next message */
- if (first_rec == NULL) {
- first_rec = rec;
- first_seq = *seq;
- }
- last_rec = rec;
- last_seq = *seq;
-
- rec = index->next(index, rec); *seq += 1;
- } while (rec != NULL);
-
- if (first_rec == NULL) {
- *seq += 1;
- *next_rec = rec == NULL ? NULL : index->next(index, rec);
- } else {
- if (!index->expunge(index, first_rec, last_rec,
- first_seq, last_seq, TRUE))
- return -1;
-
- *seq = first_seq + 1;
- *next_rec = index->lookup(index, *seq);
- }
-
- return ret;
-}
-
-static int mbox_sync_from_stream(struct mail_index *index,
- struct istream *input)
-{
- struct mail_index_record *rec;
- uoff_t from_offset;
- const unsigned char *data;
- size_t size;
- unsigned int seq;
- int dirty, ret;
-
- if (mail_cache_lock(index->cache, FALSE) <= 0)
- return -1;
- mail_cache_unlock_later(index->cache);
-
- mbox_skip_empty_lines(input);
-
- /* first make sure we start with a "From " line. If file is too
- small, we'll just treat it as empty mbox file. */
- if (i_stream_read_data(input, &data, &size, 5) > 0 &&
- memcmp(data, "From ", 5) != 0) {
- index_set_error(index, "File isn't in mbox format: %s",
- index->mailbox_path);
- return -1;
- }
-
- /* we'll go through the mailbox and index in order matching the
- messages by their size and Message-ID. old mails aren't remembered,
- so we handle well only the cases when mail has been deleted. if
- mails have been reordered (eg. sorted by someone) most of the mails
- will show up as being new. if we really wanted to support that well,
- we could save the message-ids into hash but I don't know if it's
- worth the trouble. */
-
- seq = 1;
- rec = index->lookup(index, 1);
-
- dirty = FALSE;
- while (rec != NULL) {
- from_offset = input->v_offset;
- if (input->v_offset != 0) {
- /* we're at the [\r]\n before the From-line,
- skip it */
- if (!mbox_skip_crlf(input)) {
- /* they just went and broke it, even while
- we had it locked. */
- index_set_error(index,
- "Error syncing mbox file %s: "
- "LF not found where expected",
- index->mailbox_path);
- return -1;
- }
- }
-
- ret = match_next_record(index, rec, &seq, input, &rec, &dirty);
- if (ret < 0)
- return -1;
-
- if (ret == 0) {
- /* Get back to line before From */
- i_stream_seek(input, from_offset);
- }
- }
-
- /* delete the rest of the records */
- if (rec != NULL) {
- if (!index->expunge(index, rec, INDEX_END_RECORD(index)-1,
- seq, index->header->messages_count, TRUE))
- return -1;
- }
-
- if (!dirty &&
- (index->header->flags & MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES)) {
- /* no flags are dirty anymore, no need to rewrite */
- index->header->flags &= ~MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES;
- }
-
- if ((index->set_flags & MAIL_INDEX_HDR_FLAG_REBUILD))
- return 1;
- else
- return mbox_index_append_stream(index, input);
-}
-
-int mbox_sync_full(struct mail_index *index)
-{
- struct istream *input;
- struct stat orig_st, st;
- uoff_t continue_offset;
- int ret, failed;
-
- i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
- input = mbox_get_stream(index, MAIL_LOCK_SHARED);
- if (input == NULL)
- return FALSE;
-
- if (fstat(index->mbox_fd, &orig_st) < 0) {
- mbox_set_syscall_error(index, "fstat()");
- continue_offset = (uoff_t)-1;
- failed = TRUE;
- } else {
- ret = mbox_sync_from_stream(index, input);
- failed = ret < 0;
- continue_offset = ret != 0 ||
- (index->set_flags & MAIL_INDEX_HDR_FLAG_REBUILD) ?
- (uoff_t)-1 : input->v_offset;
- i_stream_unref(input);
- }
-
- if (continue_offset != (uoff_t)-1) {
- /* mbox_index_append() stopped, which means that it wants
- write access to mbox. if mbox hasn't changed after
- unlock+lock, we should be able to safely continue where we
- were left off last time. otherwise do full resync. */
- if (!mbox_unlock(index))
- return FALSE;
-
- input = mbox_get_stream(index, MAIL_LOCK_EXCLUSIVE);
- if (input == NULL)
- return FALSE;
-
- if (fstat(index->mbox_fd, &st) < 0) {
- mbox_set_syscall_error(index, "fstat()");
- failed = TRUE;
- } else if (st.st_mtime == orig_st.st_mtime &&
- st.st_size == orig_st.st_size) {
- i_stream_seek(input, continue_offset);
- failed = mbox_index_append_stream(index, input) <= 0;
- } else {
- failed = mbox_sync_from_stream(index, input) <= 0;
- }
-
- if (index->mbox_rewritten) {
- /* rewritten, sync again */
- index->mbox_rewritten = FALSE;
- i_stream_seek(input, 0);
- failed = mbox_sync_from_stream(index, input) <= 0;
- }
-
- i_stream_unref(input);
- }
-
- return !failed;
-}
+++ /dev/null
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "mbox-index.h"
-#include "mbox-lock.h"
-#include "mail-index-util.h"
-
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-
-static int mbox_lock_and_sync_full(struct mail_index *index,
- enum mail_lock_type data_lock_type)
-{
- enum mail_lock_type lock_type;
-
- /* syncing needs exclusive index lock and shared
- mbox lock, but if we'd want exclusive mbox lock
- we need to set it here already */
- if (index->lock_type == MAIL_LOCK_SHARED)
- (void)mail_index_set_lock(index, MAIL_LOCK_UNLOCK);
-
- if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
- return FALSE;
-
- if (index->mbox_lock_type == MAIL_LOCK_UNLOCK) {
- lock_type = data_lock_type == MAIL_LOCK_EXCLUSIVE ?
- MAIL_LOCK_EXCLUSIVE : MAIL_LOCK_SHARED;
- if (!mbox_lock(index, lock_type))
- return FALSE;
- }
-
- return mbox_sync_full(index);
-}
-
-int mbox_index_sync(struct mail_index *index, int minimal_sync __attr_unused__,
- enum mail_lock_type data_lock_type, int *changes)
-{
- struct stat st;
- int count, fd;
-
- if (index->mailbox_readonly && data_lock_type == MAIL_LOCK_EXCLUSIVE) {
- index_set_error(index, "sync: %s is read-only, "
- "can't get exclusive lock",
- index->mailbox_path);
- return FALSE;
- }
-
- if (changes != NULL)
- *changes = FALSE;
-
- if (index->mbox_sync_counter == index->mbox_lock_counter) {
- /* we've already synced in this locking session */
- return TRUE;
- }
-
- i_assert(index->lock_type != MAIL_LOCK_SHARED);
-
- count = 0;
- while (stat(index->mailbox_path, &st) < 0) {
- if (errno != ENOENT || ++count == 3)
- return mbox_set_syscall_error(index, "stat()");
-
- /* mbox was deleted by someone - happens with some MUAs
- when all mail is expunged. easiest way to deal with this
- is to recreate the file. */
- fd = open(index->mailbox_path, O_RDWR | O_CREAT | O_EXCL, 0660);
- if (fd != -1)
- (void)close(fd);
- else if (errno != EEXIST)
- return mbox_set_syscall_error(index, "open()");
- }
-
- if (index->mbox_fd != -1 &&
- (index->mbox_ino != st.st_ino ||
- !CMP_DEV_T(index->mbox_dev, st.st_dev))) {
- /* mbox file was overwritten, close it if it was open */
- index->mbox_dev = st.st_dev;
- index->mbox_ino = st.st_ino;
- index->sync_size = (uoff_t)-1;
- index->sync_stamp = (time_t)-1;
-
- mbox_file_close_fd(index);
- }
-
- if (index->sync_stamp != st.st_mtime ||
- index->sync_size != (uoff_t)st.st_size) {
- mbox_file_close_stream(index);
-
- if (changes != NULL)
- *changes = TRUE;
-
- if (!mbox_lock_and_sync_full(index, data_lock_type))
- return FALSE;
-
- if ((index->set_flags & MAIL_INDEX_HDR_FLAG_REBUILD) != 0) {
- /* uidvalidity probably changed, rebuild */
- if (!index->rebuild(index))
- return FALSE;
- }
-
- if (fstat(index->mbox_fd, &st) < 0)
- return mbox_set_syscall_error(index, "fstat()");
-
- index->sync_stamp = st.st_mtime;
- index->sync_size = st.st_size;
- }
-
- /* we need some index lock to be able to lock mbox */
- if (index->lock_type == MAIL_LOCK_UNLOCK) {
- if (!index->set_lock(index, MAIL_LOCK_SHARED))
- return FALSE;
- }
-
- if (data_lock_type == MAIL_LOCK_UNLOCK) {
- if (!mbox_unlock(index))
- return FALSE;
- } else {
- if (!mbox_lock(index, data_lock_type))
- return FALSE;
- }
-
- index->mbox_sync_counter = index->mbox_lock_counter;
- return TRUE;
-}
projects / thirdparty / dovecot / core.git / commitdiff
