lib-index: Add mail_cache_close_mail() to smartly drop cached data with INDEX=MEMORY

Instead of reseting the entire transaction buffer when 256 kB is reached,
just drop mails have have been marked closed with mail_cache_close_mail().
If that's not enough, continue deleting forcibly until the buffer is below
256 kB.

This is especially important when mail_prefetch_count>0 and INDEX=MEMORY.
In that case there can be multiple mails that are being added to cache
and used later on. If they were dropped from cache too early, the work
would have to be done all over again.

diff --git a/src/lib-index/mail-cache-transaction.c b/src/lib-index/mail-cache-transaction.c
index 61018b3f2c096fe40994a9488a2f64eda2c2b4ee..96fedd457f0f1296ec688ca535c283c74eb7be3b 100644 (file)
--- a/src/lib-index/mail-cache-transaction.c
+++ b/src/lib-index/mail-cache-transaction.c
@@ -39,6 +39,7 @@ struct mail_cache_transaction_ctx {
 
        buffer_t *cache_data;
        ARRAY(struct mail_cache_transaction_rec) cache_data_seq;
+       ARRAY_TYPE(seq_range) cache_data_wanted_seqs;
        uint32_t prev_seq, min_seq;
        size_t last_rec_pos;
 
@@ -159,6 +160,8 @@ void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx **_ctx)
                buffer_free(&ctx->cache_data);
        if (array_is_created(&ctx->cache_data_seq))
                array_free(&ctx->cache_data_seq);
+       if (array_is_created(&ctx->cache_data_wanted_seqs))
+               array_free(&ctx->cache_data_wanted_seqs);
        i_free(ctx);
 }
 
@@ -463,9 +466,40 @@ mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
        ctx->min_seq = 0;
 
        array_clear(&ctx->cache_data_seq);
+       array_clear(&ctx->cache_data_wanted_seqs);
        return ret;
 }
 
+static void
+mail_cache_transaction_drop_unwanted(struct mail_cache_transaction_ctx *ctx,
+                                    size_t space_needed)
+{
+       struct mail_cache_transaction_rec *recs;
+       unsigned int i, count;
+
+       recs = array_get_modifiable(&ctx->cache_data_seq, &count);
+       /* find out how many records to delete. delete all unwanted sequences,
+          and if that's not enough delete some more. */
+       for (i = 0; i < count; i++) {
+               if (seq_range_exists(&ctx->cache_data_wanted_seqs, recs[i].seq)) {
+                       if (recs[i].cache_data_pos >= space_needed)
+                               break;
+                       /* we're going to forcibly delete it - remove it also
+                          from the array since it's no longer useful there */
+                       seq_range_array_remove(&ctx->cache_data_wanted_seqs,
+                                              recs[i].seq);
+               }
+       }
+       unsigned int deleted_count = i;
+       size_t deleted_space = i < count ?
+               recs[i].cache_data_pos : ctx->last_rec_pos;
+       for (; i < count; i++)
+               recs[i].cache_data_pos -= deleted_space;
+       ctx->last_rec_pos -= deleted_space;
+       array_delete(&ctx->cache_data_seq, 0, deleted_count);
+       buffer_delete(ctx->cache_data, 0, deleted_space);
+}
+
 static size_t
 mail_cache_transaction_update_last_rec_size(struct mail_cache_transaction_ctx *ctx)
 {
@@ -513,6 +547,7 @@ mail_cache_transaction_switch_seq(struct mail_cache_transaction_ctx *ctx)
                        buffer_create_dynamic(default_pool,
                                              MAIL_CACHE_INIT_WRITE_BUFFER);
                i_array_init(&ctx->cache_data_seq, 64);
+               i_array_init(&ctx->cache_data_wanted_seqs, 32);
        }
 
        i_zero(&new_rec);
@@ -727,6 +762,7 @@ void mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
        if (ctx->prev_seq != seq) {
                mail_cache_transaction_switch_seq(ctx);
                ctx->prev_seq = seq;
+               seq_range_array_add(&ctx->cache_data_wanted_seqs, seq);
 
                /* remember roughly what we have modified, so cache lookups can
                   look into transactions to see changes. */
@@ -750,7 +786,12 @@ void mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
                   cache file had been compressed and was reopened, return
                   without adding the cached data since cache_data buffer
                   doesn't contain the cache_rec anymore. */
-               if (mail_cache_transaction_flush(ctx) < 0) {
+               if (MAIL_INDEX_IS_IN_MEMORY(ctx->cache->index)) {
+                       /* just drop the old data to free up memory */
+                       size_t space_needed = ctx->cache_data->used +
+                               full_size - MAIL_CACHE_MAX_WRITE_BUFFER;
+                       mail_cache_transaction_drop_unwanted(ctx, space_needed);
+               } else if (mail_cache_transaction_flush(ctx) < 0) {
                        /* make sure the transaction is reset, so we don't
                           constantly try to flush for each call to this
                           function */
@@ -812,3 +853,10 @@ bool mail_cache_field_can_add(struct mail_cache_transaction_ctx *ctx,
 
        return mail_cache_field_exists(ctx->view, seq, field_idx) == 0;
 }
+
+void mail_cache_close_mail(struct mail_cache_transaction_ctx *ctx,
+                          uint32_t seq)
+{
+       if (array_is_created(&ctx->cache_data_wanted_seqs))
+               seq_range_array_remove(&ctx->cache_data_wanted_seqs, seq);
+}

diff --git a/src/lib-index/mail-cache.h b/src/lib-index/mail-cache.h
index 8986afeb5a685db4c55be67051772abf068ad4d8..2d9f4cad6bfb46a6f1fe465bb43edd3d1de31c9e 100644 (file)
--- a/src/lib-index/mail-cache.h
+++ b/src/lib-index/mail-cache.h
@@ -115,6 +115,11 @@ bool mail_cache_field_want_add(struct mail_cache_transaction_ctx *ctx,
    returned only if the decision is a forced no. */
 bool mail_cache_field_can_add(struct mail_cache_transaction_ctx *ctx,
                              uint32_t seq, unsigned int field_idx);
+/* Notify cache that the mail is now closed. Any records added with
+   mail_cache_add() are unlikely to be required again. This mainly tells
+   INDEX=MEMORY that it can free up the memory used by the mail. */
+void mail_cache_close_mail(struct mail_cache_transaction_ctx *ctx,
+                          uint32_t seq);
 
 /* Returns 1 if field exists, 0 if not, -1 if error. */
 int mail_cache_field_exists(struct mail_cache_view *view, uint32_t seq,