lib-index: mail_cache_lookup*() can now return fields recently added with mail_cache_...

Previously it was returning them if they had already been written to
dovecot.index.cache, but not if they were still in the in-memory buffer.
This avoids caching/parsing the same field multiple times when messages
aren't accessed in ascending order (e.g. when sorting messages).

diff --git a/src/lib-index/mail-cache-lookup.c b/src/lib-index/mail-cache-lookup.c
index c48dfc029b001b6b26533fac9c01b02cc59295d0..39f3eb02472c61c836747339824fa970e5b1eb16 100644 (file)
--- a/src/lib-index/mail-cache-lookup.c
+++ b/src/lib-index/mail-cache-lookup.c
@@ -163,31 +163,58 @@ void mail_cache_lookup_iter_init(struct mail_cache_view *view, uint32_t seq,
        memset(&view->loop_track, 0, sizeof(view->loop_track));
 }
 
+static bool
+mail_cache_lookup_iter_transaction(struct mail_cache_lookup_iterate_ctx *ctx)
+{
+       ctx->rec = mail_cache_transaction_lookup_rec(ctx->view->transaction,
+                                                    ctx->seq,
+                                                    &ctx->trans_next_idx);
+       if (ctx->rec == NULL)
+               return FALSE;
+
+       ctx->remap_counter = ctx->view->cache->remap_counter;
+       ctx->pos = sizeof(*ctx->rec);
+       ctx->rec_size = ctx->rec->size;
+       return TRUE;
+}
+
 static int
 mail_cache_lookup_iter_next_record(struct mail_cache_lookup_iterate_ctx *ctx)
 {
        struct mail_cache_view *view = ctx->view;
 
-       if (ctx->stop)
-               return ctx->failed ? -1 : 0;
+       if (ctx->failed)
+               return -1;
 
        if (ctx->rec != NULL)
                ctx->offset = ctx->rec->prev_offset;
        if (ctx->offset == 0) {
                /* end of this record list. check newly appended data. */
-               if (ctx->appends_checked ||
-                   view->trans_seq1 > ctx->seq ||
+               if (view->trans_seq1 > ctx->seq ||
                    view->trans_seq2 < ctx->seq ||
-                   MAIL_CACHE_IS_UNUSABLE(view->cache) ||
+                   MAIL_CACHE_IS_UNUSABLE(view->cache))
+                       return 0;
+               /* check data still in memory */
+               if (!ctx->memory_appends_checked) {
+                       if (mail_cache_lookup_iter_transaction(ctx))
+                               return 1;
+                       ctx->memory_appends_checked = TRUE;
+               }
+
+               /* check data already written to cache file */
+               if (ctx->disk_appends_checked ||
                    mail_cache_lookup_offset(view->cache, view->trans_view,
                                             ctx->seq, &ctx->offset) <= 0)
                        return 0;
 
-               ctx->appends_checked = TRUE;
+               ctx->disk_appends_checked = TRUE;
                ctx->remap_counter = view->cache->remap_counter;
                memset(&view->loop_track, 0, sizeof(view->loop_track));
        }
 
+       if (ctx->stop)
+               return 0;
+
        /* look up the next record */
        if (mail_cache_get_record(view->cache, ctx->offset, &ctx->rec) < 0)
                return -1;

diff --git a/src/lib-index/mail-cache-private.h b/src/lib-index/mail-cache-private.h
index 7b6f33cf36576ba11414811031c0b55fdfa3145f..eb573878b065746e408b042ec1d329b4fb9ff743 100644 (file)
--- a/src/lib-index/mail-cache-private.h
+++ b/src/lib-index/mail-cache-private.h
@@ -202,9 +202,12 @@ struct mail_cache_lookup_iterate_ctx {
        unsigned int pos, rec_size;
        uint32_t offset;
 
+       unsigned int trans_next_idx;
+
        unsigned int stop:1;
        unsigned int failed:1;
-       unsigned int appends_checked:1;
+       unsigned int memory_appends_checked:1;
+       unsigned int disk_appends_checked:1;
 };
 
 /* Explicitly lock the cache file. Returns -1 if error / timed out,
@@ -243,6 +246,10 @@ void mail_cache_lookup_iter_init(struct mail_cache_view *view, uint32_t seq,
 /* Returns 1 if field was returned, 0 if end of fields, or -1 if error */
 int mail_cache_lookup_iter_next(struct mail_cache_lookup_iterate_ctx *ctx,
                                struct mail_cache_iterate_field *field_r);
+const struct mail_cache_record *
+mail_cache_transaction_lookup_rec(struct mail_cache_transaction_ctx *ctx,
+                                 unsigned int seq,
+                                 unsigned int *trans_next_idx);
 
 int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size,
                   const void **data_r);

diff --git a/src/lib-index/mail-cache-transaction.c b/src/lib-index/mail-cache-transaction.c
index 6049d336d62a73389c96a9d6306a438c2e75fcb1..b56aa946d0878644ce00e2b9425286c1f3fde84a 100644 (file)
--- a/src/lib-index/mail-cache-transaction.c
+++ b/src/lib-index/mail-cache-transaction.c
@@ -21,6 +21,11 @@
 #define CACHE_TRANS_CONTEXT(obj) \
        MODULE_CONTEXT(obj, cache_mail_index_transaction_module)
 
+struct mail_cache_transaction_rec {
+       uint32_t seq;
+       uint32_t cache_data_pos;
+};
+
 struct mail_cache_transaction_ctx {
        union mail_index_transaction_module_context module_ctx;
        struct mail_index_transaction_vfuncs super;
@@ -33,7 +38,7 @@ struct mail_cache_transaction_ctx {
        uint32_t first_new_seq;
 
        buffer_t *cache_data;
-       ARRAY(uint32_t) cache_data_seq;
+       ARRAY(struct mail_cache_transaction_rec) cache_data_seq;
        uint32_t prev_seq, min_seq;
        size_t last_rec_pos;
 
@@ -272,13 +277,33 @@ static int mail_cache_transaction_lock(struct mail_cache_transaction_ctx *ctx)
        return 1;
 }
 
+const struct mail_cache_record *
+mail_cache_transaction_lookup_rec(struct mail_cache_transaction_ctx *ctx,
+                                 unsigned int seq,
+                                 unsigned int *trans_next_idx)
+{
+       const struct mail_cache_transaction_rec *recs;
+       unsigned int i, count;
+
+       recs = array_get(&ctx->cache_data_seq, &count);
+       for (i = *trans_next_idx; i < count; i++) {
+               if (recs[i].seq == seq) {
+                       *trans_next_idx = i + 1;
+                       return CONST_PTR_OFFSET(ctx->cache_data->data,
+                                               recs[i].cache_data_pos);
+               }
+       }
+       *trans_next_idx = i;
+       return NULL;
+}
+
 static int
 mail_cache_transaction_update_index(struct mail_cache_transaction_ctx *ctx,
                                    uint32_t write_offset)
 {
        struct mail_cache *cache = ctx->cache;
        const struct mail_cache_record *rec = ctx->cache_data->data;
-       const uint32_t *seqs;
+       const struct mail_cache_transaction_rec *recs;
        uint32_t i, seq_count;
 
        mail_index_ext_using_reset_id(ctx->trans, ctx->cache->ext_id,
@@ -287,9 +312,9 @@ mail_cache_transaction_update_index(struct mail_cache_transaction_ctx *ctx,
        /* write the cache_offsets to index file. records' prev_offset
           is updated to point to old cache record when index is being
           synced. */
-       seqs = array_get(&ctx->cache_data_seq, &seq_count);
+       recs = array_get(&ctx->cache_data_seq, &seq_count);
        for (i = 0; i < seq_count; i++) {
-               mail_index_update_ext(ctx->trans, seqs[i], cache->ext_id,
+               mail_index_update_ext(ctx->trans, recs[i].seq, cache->ext_id,
                                      &write_offset, NULL);
 
                write_offset += rec->size;
@@ -304,7 +329,8 @@ mail_cache_link_records(struct mail_cache_transaction_ctx *ctx,
 {
        struct mail_index_map *map;
        struct mail_cache_record *rec;
-       const uint32_t *seqs, *prev_offsetp;
+       const struct mail_cache_transaction_rec *recs;
+       const uint32_t *prev_offsetp;
        ARRAY_TYPE(uint32_t) seq_offsets;
        uint32_t i, seq_count, reset_id, prev_offset, *offsetp;
        const void *data;
@@ -312,15 +338,15 @@ mail_cache_link_records(struct mail_cache_transaction_ctx *ctx,
        i_assert(ctx->min_seq != 0);
 
        i_array_init(&seq_offsets, 64);
-       seqs = array_get(&ctx->cache_data_seq, &seq_count);
+       recs = array_get(&ctx->cache_data_seq, &seq_count);
        rec = buffer_get_modifiable_data(ctx->cache_data, NULL);
        for (i = 0; i < seq_count; i++) {
                offsetp = array_idx_modifiable(&seq_offsets,
-                                              seqs[i] - ctx->min_seq);
+                                              recs[i].seq - ctx->min_seq);
                if (*offsetp != 0)
                        prev_offset = *offsetp;
                else {
-                       mail_index_lookup_ext_full(ctx->view->trans_view, seqs[i],
+                       mail_index_lookup_ext_full(ctx->view->trans_view, recs[i].seq,
                                                   ctx->cache->ext_id, &map,
                                                   &data, NULL);
                        prev_offsetp = data;
@@ -423,6 +449,7 @@ mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
 static void
 mail_cache_transaction_update_last_rec(struct mail_cache_transaction_ctx *ctx)
 {
+       struct mail_cache_transaction_rec *trans_rec;
        struct mail_cache_record *rec;
        void *data;
        size_t size;
@@ -439,7 +466,9 @@ mail_cache_transaction_update_last_rec(struct mail_cache_transaction_ctx *ctx)
 
        if (ctx->min_seq > ctx->prev_seq || ctx->min_seq == 0)
                ctx->min_seq = ctx->prev_seq;
-       array_append(&ctx->cache_data_seq, &ctx->prev_seq, 1);
+       trans_rec = array_append_space(&ctx->cache_data_seq);
+       trans_rec->seq = ctx->prev_seq;
+       trans_rec->cache_data_pos = ctx->last_rec_pos;
        ctx->last_rec_pos = size;
 }