Cache API updates. Don't return direct pointers to mmaped cache file anymore.

--HG--
branch : HEAD

diff --git a/src/lib-index/mail-cache-compress.c b/src/lib-index/mail-cache-compress.c
index 1f296dd775cfb31fec318e917086070c054e59f7..35525448175b0cab15ca7a835719967c5c12328f 100644 (file)
--- a/src/lib-index/mail-cache-compress.c
+++ b/src/lib-index/mail-cache-compress.c
@@ -16,14 +16,15 @@ mail_cache_compress_record(struct mail_cache_view *view, uint32_t seq,
        enum mail_cache_field cached_fields, field;
        struct mail_cache_record cache_rec;
        buffer_t *buffer;
+       pool_t pool;
        const void *data;
        size_t size, pos;
-       uint32_t size32;
+       uint32_t *p, size32 = 0;
        int i;
 
        memset(&cache_rec, 0, sizeof(cache_rec));
-       buffer = buffer_create_dynamic(pool_datastack_create(),
-                                      4096, (size_t)-1);
+       pool = pool_datastack_create();
+       buffer = buffer_create_dynamic(pool, 4096, (size_t)-1);
 
        cached_fields = orig_cached_fields & ~MAIL_CACHE_HEADERS_MASK;
        buffer_append(buffer, &cache_rec, sizeof(cache_rec));
@@ -31,18 +32,24 @@ mail_cache_compress_record(struct mail_cache_view *view, uint32_t seq,
                if ((cached_fields & field) == 0)
                        continue;
 
-               if (!mail_cache_lookup_field(view, seq, field, &data, &size)) {
+               pos = buffer_get_used_size(buffer);
+               if ((field & MAIL_CACHE_FIXED_MASK) == 0)
+                       buffer_append(buffer, &size32, sizeof(size32));
+
+               if (!mail_cache_lookup_field(view, buffer, seq, field)) {
                        cached_fields &= ~field;
+                       buffer_set_used_size(buffer, pos);
                        continue;
                }
 
-               size32 = (uint32_t)size;
+               if ((field & MAIL_CACHE_FIXED_MASK) == 0) {
+                       p = buffer_get_space_unsafe(buffer, pos,
+                                                   sizeof(uint32_t));
+                       *p = (uint32_t)size;
+               }
 
-               if ((field & MAIL_CACHE_FIXED_MASK) == 0)
-                       buffer_append(buffer, &size32, sizeof(size32));
-               buffer_append(buffer, data, size);
-               if ((size32 & 3) != 0)
-                       buffer_append(buffer, null4, 4 - (size32 & 3));
+               if ((size & 3) != 0)
+                       buffer_append(buffer, null4, 4 - (size & 3));
        }
 
        /* now merge all the headers if we have them all */
@@ -53,15 +60,15 @@ mail_cache_compress_record(struct mail_cache_view *view, uint32_t seq,
 
                for (i = 0; i <= header_idx; i++) {
                        field = mail_cache_header_fields[i];
-                       if (mail_cache_lookup_field(view, seq, field,
-                                                   &data, &size) && size > 1) {
-                               size--; /* terminating \0 */
-                               buffer_append(buffer, data, size);
-                               size32 += size;
+                       if (mail_cache_lookup_field(view, buffer, seq, field)) {
+                               /* remove terminating \0 */
+                               buffer_set_used_size(buffer,
+                                       buffer_get_used_size(buffer)-1);
                        }
                }
                buffer_append(buffer, null4, 1);
-               size32++;
+
+               size32 = buffer_get_used_size(buffer) - pos;
                if ((size32 & 3) != 0)
                        buffer_append(buffer, null4, 4 - (size32 & 3));
                buffer_write(buffer, pos, &size32, sizeof(size32));

diff --git a/src/lib-index/mail-cache-lookup.c b/src/lib-index/mail-cache-lookup.c
index 00463e5191ac35e5c35015941b0084dfce1d6805..fd795b587023049bc9b76f6906c29fbf47b20aa6 100644 (file)
--- a/src/lib-index/mail-cache-lookup.c
+++ b/src/lib-index/mail-cache-lookup.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "buffer.h"
+#include "str.h"
 #include "mail-cache-private.h"
 
 #define CACHE_PREFETCH 1024
@@ -204,8 +205,7 @@ mail_cache_get_fields(struct mail_cache_view *view, uint32_t seq)
 
 static int cache_get_field(struct mail_cache *cache,
                           const struct mail_cache_record *cache_rec,
-                          enum mail_cache_field field,
-                          const void **data_r, size_t *size_r)
+                          enum mail_cache_field field, buffer_t *dest_buf)
 {
        unsigned int mask;
        uint32_t data_size;
@@ -247,8 +247,9 @@ static int cache_get_field(struct mail_cache *cache,
                                                         "Field size is 0");
                                return FALSE;
                        }
-                       *data_r = CONST_PTR_OFFSET(cache_rec, offset);
-                       *size_r = data_size;
+                       buffer_append(dest_buf,
+                                     CONST_PTR_OFFSET(cache_rec, offset),
+                                     data_size);
                        return TRUE;
                }
                offset = prev_offset;
@@ -258,9 +259,8 @@ static int cache_get_field(struct mail_cache *cache,
        return FALSE;
 }
 
-int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq,
-                           enum mail_cache_field field,
-                           const void **data_r, size_t *size_r)
+int mail_cache_lookup_field(struct mail_cache_view *view, buffer_t *dest_buf,
+                           uint32_t seq, enum mail_cache_field field)
 {
        struct mail_cache_record *cache_rec;
 
@@ -270,7 +270,7 @@ int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq,
        while (cache_rec != NULL) {
                if ((cache_rec->fields & field) != 0) {
                        return cache_get_field(view->cache, cache_rec, field,
-                                              data_r, size_r);
+                                              dest_buf);
                }
                cache_rec = mail_cache_get_record(view->cache,
                                                  cache_rec->prev_offset);
@@ -279,45 +279,25 @@ int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq,
        return FALSE;
 }
 
-const char *
-mail_cache_lookup_string_field(struct mail_cache_view *view, uint32_t seq,
-                              enum mail_cache_field field)
+int mail_cache_lookup_string_field(struct mail_cache_view *view, string_t *dest,
+                                  uint32_t seq, enum mail_cache_field field)
 {
-       const void *data;
-       size_t size;
+       size_t old_size, new_size;
 
        i_assert((field & MAIL_CACHE_STRING_MASK) != 0);
 
-       if (!mail_cache_lookup_field(view, seq, field, &data, &size))
-               return NULL;
+       old_size = str_len(dest);
+       if (!mail_cache_lookup_field(view, dest, seq, field))
+               return FALSE;
 
-       if (((const char *)data)[size-1] != '\0') {
+       new_size = str_len(dest);
+       if (old_size == new_size ||
+           str_data(dest)[new_size-1] != '\0') {
                mail_cache_set_corrupted(view->cache,
                        "String field %x doesn't end with NUL", field);
-               return NULL;
-       }
-       return data;
-}
-
-int mail_cache_copy_fixed_field(struct mail_cache_view *view, uint32_t seq,
-                               enum mail_cache_field field,
-                               void *buffer, size_t buffer_size)
-{
-       const void *data;
-       size_t size;
-
-       i_assert((field & MAIL_CACHE_FIXED_MASK) != 0);
-
-       if (!mail_cache_lookup_field(view, seq, field, &data, &size))
                return FALSE;
-
-       if (buffer_size != size) {
-               i_panic("cache: fixed field %x wrong size "
-                       "(%"PRIuSIZE_T" vs %"PRIuSIZE_T")",
-                       field, size, buffer_size);
        }
-
-       memcpy(buffer, data, buffer_size);
+       str_truncate(dest, new_size-1);
        return TRUE;
 }
 

diff --git a/src/lib-index/mail-cache.h b/src/lib-index/mail-cache.h
index 7308444d9205cec975d42c555093654b0a5781dd..5abfa440c3afe15575088c0445c8f6e91eb4cf9a 100644 (file)
--- a/src/lib-index/mail-cache.h
+++ b/src/lib-index/mail-cache.h
@@ -113,21 +113,12 @@ mail_cache_get_fields(struct mail_cache_view *view, uint32_t seq);
 /* Set data_r and size_r to point to wanted field in cache file.
    Returns TRUE if field was found. If field contains multiple fields,
    first one found is returned. This is mostly useful for finding headers. */
-int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq,
-                           enum mail_cache_field field,
-                           const void **data_r, size_t *size_r);
+int mail_cache_lookup_field(struct mail_cache_view *view, buffer_t *dest_buf,
+                           uint32_t seq, enum mail_cache_field field);
 
 /* Return string field. */
-const char *
-mail_cache_lookup_string_field(struct mail_cache_view *view, uint32_t seq,
-                              enum mail_cache_field field);
-
-/* Copy fixed size field to given buffer. buffer_size must be exactly the
-   expected size. The result will be converted to host byte order.
-   Returns TRUE if field was found. */
-int mail_cache_copy_fixed_field(struct mail_cache_view *view, uint32_t seq,
-                               enum mail_cache_field field,
-                               void *buffer, size_t buffer_size);
+int mail_cache_lookup_string_field(struct mail_cache_view *view, string_t *dest,
+                                  uint32_t seq, enum mail_cache_field field);
 
 /* Mark given field as missing, ie. it should be cached when possible. */
 void mail_cache_mark_missing(struct mail_cache_view *view, uint32_t uid,

diff --git a/src/lib-storage/index/index-mail-headers.c b/src/lib-storage/index/index-mail-headers.c
index 54c4f947e80ce6d217eff0b114e7b4d381887929..c89242b0769eeb4fcd2cef81b7fa2cc25d46d639 100644 (file)
--- a/src/lib-storage/index/index-mail-headers.c
+++ b/src/lib-storage/index/index-mail-headers.c
@@ -419,30 +419,29 @@ static int parse_cached_headers(struct index_mail *mail, int idx)
        struct message_header_parser_ctx *hdr_ctx;
        struct message_header_line *hdr;
        struct istream *istream;
-       const char *str, *const *idx_headers;
+       const char *const *idx_headers;
+       string_t *str;
 
-       t_push();
        if (idx < data->header_data_cached) {
                /* it's already in header_data. */
-               istream = i_stream_create_from_data(pool_datastack_create(),
+               istream = i_stream_create_from_data(mail->pool,
                                                    str_data(data->header_data),
                                                    str_len(data->header_data));
                /* we might be parsing a bit more.. */
                idx = data->header_data_cached-1;
                data->header_stream = istream;
        } else {
-               str = mail_cache_lookup_string_field(
-                       mail->trans->cache_view, data->seq,
-                       mail_cache_header_fields[idx]);
-               if (str == NULL) {
+               str = str_new(mail->pool, 32);
+               if (!mail_cache_lookup_string_field(
+                               mail->trans->cache_view, str, data->seq,
+                               mail_cache_header_fields[idx])) {
                        /* broken - we expected the header to exist */
-                       t_pop();
                        return FALSE;
                }
 
                data->header_data_cached_partial = TRUE;
-               istream = i_stream_create_from_data(pool_datastack_create(),
-                                                   str, strlen(str));
+               istream = i_stream_create_from_data(mail->pool, str_data(str),
+                                                   str_len(str));
        }
 
        idx_headers = mail_cache_get_header_fields(mail->trans->cache_view,
@@ -450,7 +449,6 @@ static int parse_cached_headers(struct index_mail *mail, int idx)
        if (idx_headers == NULL) {
                mail_cache_set_corrupted(mail->ibox->cache,
                        "Headers %d names not found", idx);
-               t_pop();
                return FALSE;
        }
 
@@ -466,7 +464,6 @@ static int parse_cached_headers(struct index_mail *mail, int idx)
 
        data->header_stream = NULL;
        i_stream_unref(istream);
-       t_pop();
 
        return TRUE;
 }
@@ -485,7 +482,7 @@ static void trash_partial_headers(struct index_mail *mail)
 int index_mail_parse_headers(struct index_mail *mail)
 {
        struct index_mail_data *data = &mail->data;
-       const char *str, *const *headers;
+       const char *const *headers;
        int idx, max;
 
        if (data->stream == NULL) {
@@ -508,14 +505,12 @@ int index_mail_parse_headers(struct index_mail *mail)
                /* add all cached headers to beginning of header_data */
                 idx = data->header_data_cached; max = idx-1;
                for (; idx < MAIL_CACHE_HEADERS_COUNT; idx++) {
-                       str = mail_cache_lookup_string_field(
-                               mail->trans->cache_view, mail->data.seq,
-                               mail_cache_header_fields[idx]);
-                       if (str == NULL)
+                       if (!mail_cache_lookup_string_field(
+                               mail->trans->cache_view, mail->data.header_data,
+                               mail->data.seq, mail_cache_header_fields[idx]))
                                continue;
 
                        max = idx;
-                       str_append(mail->data.header_data, str);
                }
                data->header_data_cached = max+1;
                data->header_data_uncached_offset =
@@ -635,7 +630,7 @@ struct istream *index_mail_get_headers(struct mail *_mail,
        struct index_mail *mail = (struct index_mail *) _mail;
        struct index_mail_data *data = &mail->data;
        struct cached_header *hdr;
-       const char *const *tmp, *str;
+       const char *const *tmp;
        int i, idx, all_saved;
 
        i_assert(*minimum_fields != NULL);
@@ -652,13 +647,9 @@ struct istream *index_mail_get_headers(struct mail *_mail,
                        trash_partial_headers(mail);
                }
                for (i = data->header_data_cached; i <= idx; i++) {
-                       str = mail_cache_lookup_string_field(
-                                       mail->trans->cache_view, data->seq,
-                                       mail_cache_header_fields[i]);
-                       if (str == NULL)
-                               continue;
-
-                       str_append(data->header_data, str);
+                       (void)mail_cache_lookup_string_field(
+                               mail->trans->cache_view, data->header_data,
+                               data->seq, mail_cache_header_fields[i]);
                }
                data->header_data_cached = idx+1;
                data->header_data_uncached_offset = str_len(data->header_data);

diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c
index 22bdfe8eeb3325946f0923e9031e157e432cd37d..0e4af853d4cc57fa1a5eb652bde716ee10e6f2ed 100644 (file)
--- a/src/lib-storage/index/index-mail.c
+++ b/src/lib-storage/index/index-mail.c
@@ -18,9 +18,8 @@ static void index_mail_parse_body(struct index_mail *mail);
 static struct message_part *get_cached_parts(struct index_mail *mail)
 {
        struct message_part *part;
-       const void *part_data;
+       buffer_t *part_buf;
        const char *error;
-       size_t part_size;
 
        if ((mail->data.cached_fields & MAIL_CACHE_MESSAGEPART) == 0) {
                mail_cache_mark_missing(mail->trans->cache_view, mail->data.seq,
@@ -28,14 +27,17 @@ static struct message_part *get_cached_parts(struct index_mail *mail)
                return NULL;
        }
 
-       if (!mail_cache_lookup_field(mail->trans->cache_view, mail->data.seq,
-                                    MAIL_CACHE_MESSAGEPART,
-                                    &part_data, &part_size)) {
+       part_buf = buffer_create_dynamic(pool_datastack_create(),
+                                        128, (size_t)-1);
+       if (!mail_cache_lookup_field(mail->trans->cache_view, part_buf,
+                                    mail->data.seq, MAIL_CACHE_MESSAGEPART)) {
                /* unexpected - must be an error */
                return NULL;
        }
 
-       part = message_part_deserialize(mail->pool, part_data, part_size,
+       part = message_part_deserialize(mail->pool,
+                                       buffer_get_data(part_buf, NULL),
+                                       buffer_get_used_size(part_buf),
                                        &error);
        if (part == NULL) {
                mail_cache_set_corrupted(mail->ibox->cache,
@@ -55,10 +57,10 @@ static struct message_part *get_cached_parts(struct index_mail *mail)
        return part;
 }
 
-char *index_mail_get_cached_string(struct index_mail *mail,
-                                  enum mail_cache_field field)
+const char *index_mail_get_cached_string(struct index_mail *mail,
+                                        enum mail_cache_field field)
 {
-       const char *ret;
+       string_t *str;
 
        if ((mail->data.cached_fields & field) == 0) {
                mail_cache_mark_missing(mail->trans->cache_view,
@@ -66,9 +68,35 @@ char *index_mail_get_cached_string(struct index_mail *mail,
                return NULL;
        }
 
-       ret = mail_cache_lookup_string_field(mail->trans->cache_view,
-                                            mail->data.seq, field);
-       return p_strdup(mail->pool, ret);
+       str = str_new(mail->pool, 32);
+       if (!mail_cache_lookup_string_field(mail->trans->cache_view, str,
+                                           mail->data.seq, field))
+               return NULL;
+
+       return str_c(str);
+}
+
+static int index_mail_get_fixed_field(struct index_mail *mail,
+                                     enum mail_cache_field field,
+                                     void *data, size_t data_size)
+{
+       buffer_t *buf;
+       int ret;
+
+       t_push();
+       buf = buffer_create_data(pool_datastack_create(), data, data_size);
+       if (!mail_cache_lookup_field(mail->trans->cache_view, buf,
+                                    mail->data.seq, field)) {
+               mail_cache_mark_missing(mail->trans->cache_view,
+                                       mail->data.seq, field);
+               ret = FALSE;
+       } else {
+               i_assert(buffer_get_used_size(buf) == data_size);
+               ret = TRUE;
+       }
+       t_pop();
+
+       return ret;
 }
 
 uoff_t index_mail_get_cached_uoff_t(struct index_mail *mail,
@@ -76,13 +104,8 @@ uoff_t index_mail_get_cached_uoff_t(struct index_mail *mail,
 {
        uoff_t uoff;
 
-       if (!mail_cache_copy_fixed_field(mail->trans->cache_view,
-                                        mail->data.seq, field,
-                                        &uoff, sizeof(uoff))) {
-               mail_cache_mark_missing(mail->trans->cache_view,
-                                       mail->data.seq, field);
+       if (!index_mail_get_fixed_field(mail, field, &uoff, sizeof(uoff)))
                uoff = (uoff_t)-1;
-       }
 
        return uoff;
 }
@@ -96,28 +119,17 @@ time_t index_mail_get_cached_received_date(struct index_mail *mail)
 {
        time_t t;
 
-       if (!mail_cache_copy_fixed_field(mail->trans->cache_view,
-                                        mail->data.seq,
-                                        MAIL_CACHE_RECEIVED_DATE,
-                                        &t, sizeof(t))) {
-               mail_cache_mark_missing(mail->trans->cache_view, mail->data.seq,
-                                       MAIL_CACHE_RECEIVED_DATE);
+       if (!index_mail_get_fixed_field(mail, MAIL_CACHE_RECEIVED_DATE,
+                                       &t, sizeof(t)))
                t = (time_t)-1;
-       }
-
        return t;
 }
 
 static void get_cached_sent_date(struct index_mail *mail,
                                 struct mail_sent_date *sent_date)
 {
-       if (!mail_cache_copy_fixed_field(mail->trans->cache_view,
-                                        mail->data.seq,
-                                        MAIL_CACHE_SENT_DATE,
-                                        sent_date, sizeof(*sent_date))) {
-               mail_cache_mark_missing(mail->trans->cache_view, mail->data.seq,
-                                       MAIL_CACHE_SENT_DATE);
-
+       if (!index_mail_get_fixed_field(mail, MAIL_CACHE_SENT_DATE,
+                                       sent_date, sizeof(*sent_date))) {
                sent_date->time = (time_t)-1;
                sent_date->timezone = 0;
        }

diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h
index 4ba641147bb204dd39f3ec8ee5d1ab8f32bcf324..c2e8ac4a62db4754d305b0db288a7e7109d3072d 100644 (file)
--- a/src/lib-storage/index/index-mail.h
+++ b/src/lib-storage/index/index-mail.h
@@ -109,8 +109,8 @@ int index_mail_update_flags(struct mail *mail,
                            enum modify_type modify_type);
 int index_mail_expunge(struct mail *mail);
 
-char *index_mail_get_cached_string(struct index_mail *mail,
-                                  enum mail_cache_field field);
+const char *index_mail_get_cached_string(struct index_mail *mail,
+                                        enum mail_cache_field field);
 uoff_t index_mail_get_cached_uoff_t(struct index_mail *mail,
                                    enum mail_cache_field field);
 uoff_t index_mail_get_cached_virtual_size(struct index_mail *mail);