lib-index: Fix cache being purged too often when it had unaccessed fields

Cache was being purged when it had a field that was last accessed
after mail_cache_unaccessed_field_drop but before
2*mail_cache_unaccessed_field_drop. This purging may not have even
done anything.

Use shared code now between the check in mail_cache_header_fields_read()
and the actual purging in mail_cache_purge_check_field(). This way they
can't become desynced again.

diff --git a/src/lib-index/mail-cache-fields.c b/src/lib-index/mail-cache-fields.c
index 7638eea6882cd8efd4782b062a428acdcad01f3f..e217b2c29073c7b5938072130c28ba19fee6566d 100644 (file)
--- a/src/lib-index/mail-cache-fields.c
+++ b/src/lib-index/mail-cache-fields.c
@@ -327,8 +327,7 @@ int mail_cache_header_fields_read(struct mail_cache *cache)
        char *orig_key;
        void *orig_value;
        unsigned int fidx, new_fields_count;
-       enum mail_cache_decision_type dec;
-       time_t max_drop_time;
+       struct mail_cache_purge_drop_ctx drop_ctx;
        uint32_t offset, i;
 
        if (mail_cache_header_fields_get_offset(cache, &offset, &field_hdr) < 0)
@@ -373,10 +372,7 @@ int mail_cache_header_fields_read(struct mail_cache *cache)
        for (i = 0; i < cache->fields_count; i++)
                cache->field_file_map[i] = (uint32_t)-1;
 
-       max_drop_time = cache->index->map->hdr.day_stamp == 0 ? 0 :
-               cache->index->map->hdr.day_stamp -
-               cache->index->optimization_set.cache.unaccessed_field_drop_secs;
-
+       mail_cache_purge_drop_init(cache, &cache->index->map->hdr, &drop_ctx);
        i_zero(&field);
        for (i = 0; i < field_hdr->fields_count; i++) {
                for (p = names; p != end && *p != '\0'; p++) ;
@@ -445,17 +441,24 @@ int mail_cache_header_fields_read(struct mail_cache *cache)
                if ((time_t)last_used[i] > cache->fields[fidx].field.last_used)
                        cache->fields[fidx].field.last_used = last_used[i];
 
-               dec = cache->fields[fidx].field.decision;
-               if (cache->fields[fidx].field.last_used < max_drop_time &&
-                   cache->fields[fidx].field.last_used != 0 &&
-                   (dec & MAIL_CACHE_DECISION_FORCED) == 0 &&
-                   dec != MAIL_CACHE_DECISION_NO) {
-                       /* time to drop this field. don't bother dropping
-                          fields that have never been used. */
+               switch (mail_cache_purge_drop_test(&drop_ctx, fidx)) {
+               case MAIL_CACHE_PURGE_DROP_DECISION_NONE:
+                       break;
+               case MAIL_CACHE_PURGE_DROP_DECISION_DROP:
                        mail_cache_purge_later(cache, t_strdup_printf(
                                "Drop old field %s (last_used=%"PRIdTIME_T")",
                                cache->fields[fidx].field.name,
                                cache->fields[fidx].field.last_used));
+                       break;
+               case MAIL_CACHE_PURGE_DROP_DECISION_TO_TEMP:
+                       /* This cache decision change can cause the field to be
+                          dropped for old mails, so do it via purging. */
+                       mail_cache_purge_later(cache, t_strdup_printf(
+                               "Change cache decision to temp for old field %s "
+                               "(last_used=%"PRIdTIME_T")",
+                               cache->fields[fidx].field.name,
+                               cache->fields[fidx].field.last_used));
+                       break;
                }
 
                 names = p + 1;

diff --git a/src/lib-index/mail-cache-private.h b/src/lib-index/mail-cache-private.h
index 8dfc97f59ae467e57bb67ac40a75ba0185f4aebb..6ad14559b198d6b53ff5b8408520eff141bf3fe3 100644 (file)
--- a/src/lib-index/mail-cache-private.h
+++ b/src/lib-index/mail-cache-private.h
@@ -277,6 +277,23 @@ struct event_passthrough *
 mail_cache_decision_changed_event(struct mail_cache *cache, struct event *event,
                                  unsigned int field);
 
+struct mail_cache_purge_drop_ctx {
+       struct mail_cache *cache;
+       time_t max_yes_downgrade_time;
+       time_t max_temp_drop_time;
+};
+enum mail_cache_purge_drop_decision {
+       MAIL_CACHE_PURGE_DROP_DECISION_NONE,
+       MAIL_CACHE_PURGE_DROP_DECISION_DROP,
+       MAIL_CACHE_PURGE_DROP_DECISION_TO_TEMP,
+};
+void mail_cache_purge_drop_init(struct mail_cache *cache,
+                               const struct mail_index_header *hdr,
+                               struct mail_cache_purge_drop_ctx *ctx_r);
+enum mail_cache_purge_drop_decision
+mail_cache_purge_drop_test(struct mail_cache_purge_drop_ctx *ctx,
+                          unsigned int field);
+
 int mail_cache_expunge_handler(struct mail_index_sync_map_ctx *sync_ctx,
                               uint32_t seq, const void *data,
                               void **sync_context, void *context);

diff --git a/src/lib-index/mail-cache-purge.c b/src/lib-index/mail-cache-purge.c
index 8d0a7d9aea9fb55fb3d67ec9ac0fa1e81906686d..bb5a513ebf2ca999cd5a2b64c6c8bb875320b3a8 100644 (file)
--- a/src/lib-index/mail-cache-purge.c
+++ b/src/lib-index/mail-cache-purge.c
@@ -16,7 +16,7 @@
 struct mail_cache_copy_context {
        struct mail_cache *cache;
        struct event *event;
-       time_t max_temp_drop_time, max_yes_downgrade_time;
+       struct mail_cache_purge_drop_ctx drop_ctx;
 
        buffer_t *buffer, *field_seen;
        ARRAY(unsigned int) bitmask_pos;
@@ -157,12 +157,10 @@ mail_cache_purge_check_field(struct mail_cache_copy_context *ctx,
        struct mail_cache_field_private *priv = &ctx->cache->fields[field];
        enum mail_cache_decision_type dec = priv->field.decision;
 
-       if ((dec & MAIL_CACHE_DECISION_FORCED) != 0)
-               ;
-       else if (dec != MAIL_CACHE_DECISION_NO &&
-                priv->field.last_used < ctx->max_temp_drop_time) {
-               /* YES or TEMP decision field hasn't been accessed for a long
-                  time now. Drop it. */
+       switch (mail_cache_purge_drop_test(&ctx->drop_ctx, field)) {
+       case MAIL_CACHE_PURGE_DROP_DECISION_NONE:
+               break;
+       case MAIL_CACHE_PURGE_DROP_DECISION_DROP: {
                const char *dec_str = mail_cache_decision_to_string(dec);
                struct event_passthrough *e =
                        event_create_passthrough(ctx->event)->
@@ -174,10 +172,9 @@ mail_cache_purge_check_field(struct mail_cache_copy_context *ctx,
                        "(decision=%s, last_used=%"PRIdTIME_T")",
                        priv->field.name, dec_str, priv->field.last_used);
                dec = MAIL_CACHE_DECISION_NO;
-       } else if (dec == MAIL_CACHE_DECISION_YES &&
-                  priv->field.last_used < ctx->max_yes_downgrade_time) {
-               /* YES decision field hasn't been accessed for a while
-                  now. Change its decision to TEMP. */
+               break;
+       }
+       case MAIL_CACHE_PURGE_DROP_DECISION_TO_TEMP: {
                struct event_passthrough *e =
                        mail_cache_decision_changed_event(
                                ctx->cache, ctx->event, field)->
@@ -188,6 +185,8 @@ mail_cache_purge_check_field(struct mail_cache_copy_context *ctx,
                        "(last_used=%"PRIdTIME_T")",
                        priv->field.name, priv->field.last_used);
                dec = MAIL_CACHE_DECISION_TEMP;
+               break;
+       }
        }
        priv->field.decision = dec;
 
@@ -255,12 +254,7 @@ mail_cache_copy(struct mail_cache *cache, struct mail_index_transaction *trans,
        /* @UNSAFE: drop unused fields and create a field mapping for
           used fields */
        idx_hdr = mail_index_get_header(view);
-       if (idx_hdr->day_stamp != 0) {
-               ctx.max_yes_downgrade_time = idx_hdr->day_stamp -
-                       cache->index->optimization_set.cache.unaccessed_field_drop_secs;
-               ctx.max_temp_drop_time = idx_hdr->day_stamp -
-                       2 * cache->index->optimization_set.cache.unaccessed_field_drop_secs;
-       }
+       mail_cache_purge_drop_init(cache, idx_hdr, &ctx.drop_ctx);
 
        orig_fields_count = cache->fields_count;
        if (cache->file_fields_count == 0) {
@@ -668,3 +662,43 @@ void mail_cache_purge_later_reset(struct mail_cache *cache)
        cache->need_purge_file_seq = 0;
        i_free(cache->need_purge_reason);
 }
+
+void mail_cache_purge_drop_init(struct mail_cache *cache,
+                               const struct mail_index_header *hdr,
+                               struct mail_cache_purge_drop_ctx *ctx_r)
+{
+       i_zero(ctx_r);
+       ctx_r->cache = cache;
+       if (hdr->day_stamp != 0) {
+               const struct mail_index_cache_optimization_settings *opt =
+                       &cache->index->optimization_set.cache;
+               ctx_r->max_yes_downgrade_time = hdr->day_stamp -
+                       opt->unaccessed_field_drop_secs;
+               ctx_r->max_temp_drop_time = hdr->day_stamp -
+                       2 * opt->unaccessed_field_drop_secs;
+       }
+}
+
+enum mail_cache_purge_drop_decision
+mail_cache_purge_drop_test(struct mail_cache_purge_drop_ctx *ctx,
+                          unsigned int field)
+{
+       struct mail_cache_field_private *priv = &ctx->cache->fields[field];
+       enum mail_cache_decision_type dec = priv->field.decision;
+
+       if ((dec & MAIL_CACHE_DECISION_FORCED) != 0)
+               return MAIL_CACHE_PURGE_DROP_DECISION_NONE;
+       if (dec != MAIL_CACHE_DECISION_NO &&
+           priv->field.last_used < ctx->max_temp_drop_time) {
+               /* YES or TEMP decision field hasn't been accessed for a long
+                  time now. Drop it. */
+               return MAIL_CACHE_PURGE_DROP_DECISION_DROP;
+       }
+       if (dec == MAIL_CACHE_DECISION_YES &&
+           priv->field.last_used < ctx->max_yes_downgrade_time) {
+               /* YES decision field hasn't been accessed for a while
+                  now. Change its decision to TEMP. */
+               return MAIL_CACHE_PURGE_DROP_DECISION_TO_TEMP;
+       }
+       return MAIL_CACHE_PURGE_DROP_DECISION_NONE;
+}