global: Add mail_cache_max_headers_count functionality

diff --git a/src/lib-index/mail-cache-decisions.c b/src/lib-index/mail-cache-decisions.c
index 4c26912bca856b21bb7836d7018d6d278a251274..7d8baf7bf177e32eb726ed611277b6af106931e5 100644 (file)
--- a/src/lib-index/mail-cache-decisions.c
+++ b/src/lib-index/mail-cache-decisions.c
@@ -173,8 +173,44 @@ void mail_cache_decision_state_update(struct mail_cache_view *view,
        }
 }
 
+static unsigned int mail_cache_count_alive_headers(struct mail_cache *cache)
+{
+       unsigned int count = 0;
+       for (unsigned int index = 0; index < cache->fields_count; ++index) {
+               if (cache->fields[index].field.type == MAIL_CACHE_FIELD_HEADER &&
+                   (cache->fields[index].field.decision &
+                    ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) !=
+                                               MAIL_CACHE_DECISION_NO)
+                       ++count;
+       }
+       return count;
+}
+
+bool mail_cache_headers_check_capped(struct mail_cache *cache)
+{
+       struct mail_index_cache_optimization_settings *set =
+               &cache->index->optimization_set.cache;
+
+       if (set->max_headers_count == 0) return FALSE;
+       if (cache->headers_capped) return TRUE;
+
+       unsigned int count = mail_cache_count_alive_headers(cache);
+       cache->headers_capped = count >= set->max_headers_count;
+       return cache->headers_capped;
+}
+
+static struct event_passthrough *
+mail_cache_decision_rejected_event(struct mail_cache *cache, unsigned int field,
+                                  const char *reason)
+{
+       return event_create_passthrough(cache->event)->
+               set_name("mail_cache_decision_rejected")->
+               add_str("field", cache->fields[field].field.name)->
+               add_str("reason", reason);
+}
+
 void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq,
-                            unsigned int field)
+                            unsigned int field, bool *rejected_r)
 {
        struct mail_cache *cache = view->cache;
        struct mail_cache_field_private *priv;
@@ -182,6 +218,8 @@ void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq,
 
        i_assert(field < cache->fields_count);
 
+       *rejected_r = FALSE;
+
        if (view->no_decision_updates)
                return;
 
@@ -194,8 +232,22 @@ void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq,
        }
 
        /* field used the first time */
-       if (priv->field.decision == MAIL_CACHE_DECISION_NO)
+       if (priv->field.decision == MAIL_CACHE_DECISION_NO) {
+               if (mail_cache_headers_check_capped(view->cache)) {
+                       *rejected_r = TRUE;
+
+                       const char *reason = "too_many_headers";
+                       struct event_passthrough *e =
+                               mail_cache_decision_rejected_event(
+                                       cache, field, reason);
+                       e_debug(e->event(),
+                               "Cache rejected header '%s': %s",
+                               priv->field.name, reason);
+                       return;
+               }
+
                priv->field.decision = MAIL_CACHE_DECISION_TEMP;
+       }
        priv->field.last_used = ioloop_time;
        priv->decision_dirty = TRUE;
        cache->field_header_write_pending = TRUE;

diff --git a/src/lib-index/mail-cache-fields.c b/src/lib-index/mail-cache-fields.c
index 429e0d234cdc89c54abb69d81470be1fca53c5de..9a2f2c1af812f7b03106c7a2abd8de4b28365c5c 100644 (file)
--- a/src/lib-index/mail-cache-fields.c
+++ b/src/lib-index/mail-cache-fields.c
@@ -78,6 +78,12 @@ mail_cache_field_update(struct mail_cache *cache,
           internal fields and for mail_*cache_fields settings? */
        initial_registering = cache->file_fields_count == 0;
 
+       if (newfield->decision == MAIL_CACHE_DECISION_NO) {
+               /* reset the cached flag, it will be set again anyway on first
+                  access if that is the case */
+               cache->headers_capped = FALSE;
+       }
+
        orig = &cache->fields[newfield->idx];
        if ((newfield->decision & MAIL_CACHE_DECISION_FORCED) != 0 ||
            ((orig->field.decision & MAIL_CACHE_DECISION_FORCED) == 0 &&
@@ -346,6 +352,10 @@ int mail_cache_header_fields_read(struct mail_cache *cache)
                return -1;
        }
 
+       /* reset the cached flag, it will be set again anyway on first access if
+          that is the case */
+       cache->headers_capped = FALSE;
+
        new_fields_count = field_hdr->fields_count;
        if (new_fields_count != 0) {
                cache->file_field_map =

diff --git a/src/lib-index/mail-cache-private.h b/src/lib-index/mail-cache-private.h
index c2fee1761218ade47360fefd663f4874361064be..bb11a345bb2b3be3502a7d3b364493a1e4b7933f 100644 (file)
--- a/src/lib-index/mail-cache-private.h
+++ b/src/lib-index/mail-cache-private.h
@@ -219,6 +219,8 @@ struct mail_cache {
           larger parts of it). This is used with MAIL_INDEX_OPEN_FLAG_SAVEONLY
           to avoid unnecessary cache reads. */
        bool map_with_read:1;
+       /* Cache headers count has been capped */
+       bool headers_capped:1;
 };
 
 struct mail_cache_loop_track {
@@ -395,6 +397,8 @@ struct event_passthrough *
 mail_cache_decision_changed_event(struct mail_cache *cache, struct event *event,
                                  unsigned int field);
 
+bool mail_cache_headers_check_capped(struct mail_cache *cache);
+
 struct mail_cache_purge_drop_ctx {
        struct mail_cache *cache;
        time_t max_yes_downgrade_time;

diff --git a/src/lib-index/mail-cache-transaction.c b/src/lib-index/mail-cache-transaction.c
index 9d2c47a4d21cd9583cbdbe98f2a9f2f5cb00259f..b766da1fa2dee0c211494892816b0299e83cc8c0 100644 (file)
--- a/src/lib-index/mail-cache-transaction.c
+++ b/src/lib-index/mail-cache-transaction.c
@@ -795,7 +795,12 @@ void mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
           read. */
        mail_cache_transaction_refresh_decisions(ctx);
 
-       mail_cache_decision_add(ctx->view, seq, field_idx);
+       /* The decision to reject a field may come last minute during actual add,
+          for example if the header count limit has been reached */
+       bool rejected;
+       mail_cache_decision_add(ctx->view, seq, field_idx, &rejected);
+       if (rejected)
+               return;
 
        fixed_size = ctx->cache->fields[field_idx].field.field_size;
        i_assert(fixed_size == UINT_MAX || fixed_size == data_size);

diff --git a/src/lib-index/mail-cache.h b/src/lib-index/mail-cache.h
index 09fde2941fbe6d6918128bdd84886ed7089ef710..3a0c3db1b1303e5f9f851dc75c42b012dcf50d7b 100644 (file)
--- a/src/lib-index/mail-cache.h
+++ b/src/lib-index/mail-cache.h
@@ -162,9 +162,10 @@ enum mail_cache_decision_type
 mail_cache_field_get_decision(struct mail_cache *cache, unsigned int field_idx);
 /* Notify the decision handling code when field is committed to cache.
    If this is the first time the field is added to cache, its caching decision
-   is updated to TEMP. */
+   is updated to TEMP. Sets rejected to true if header count limit has been
+   reached and the field has NOT been switched to TEMP */
 void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq,
-                            unsigned int field);
+                            unsigned int field, bool *rejected_r);
 
 /* Set data_r and size_r to point to wanted field in cache file.
    Returns 1 if field was found, 0 if not, -1 if error. */

diff --git a/src/lib-index/mail-index.c b/src/lib-index/mail-index.c
index c0e6a4fe0621e63948d5196e32744108752d82a4..01bb93348dba236bedd444314b0c8ca722feab18 100644 (file)
--- a/src/lib-index/mail-index.c
+++ b/src/lib-index/mail-index.c
@@ -226,6 +226,8 @@ void mail_index_set_optimization_settings(struct mail_index *index,
                        set->cache.purge_header_continue_count;
        if (set->cache.record_max_size != 0)
                dest->cache.record_max_size = set->cache.record_max_size;
+
+       dest->cache.max_headers_count = set->cache.max_headers_count;
 }
 
 void mail_index_set_ext_init_data(struct mail_index *index, uint32_t ext_id,

diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h
index 6d768e050b34a19afd756b6808462ca2064570b9..c401fc679e447e248f82acf1d20dd7185ec146fb 100644 (file)
--- a/src/lib-index/mail-index.h
+++ b/src/lib-index/mail-index.h
@@ -331,6 +331,9 @@ struct mail_index_cache_optimization_settings {
        /* If cache record becomes larger than this, don't add it. */
        unsigned int record_max_size;
 
+       /* Maximum number of headers to cache */
+       unsigned int max_headers_count;
+
        /* Maximum size for the cache file. Internally the limit is 1 GB. */
        uoff_t max_size;
        /* Never purge the file if it's smaller than this */

diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c
index 12079a1751d91e11279c2ddb5475e3f4712125dc..5a2d45398e103894aa5d0346da768df9520eb51f 100644 (file)
--- a/src/lib-storage/index/index-mail.c
+++ b/src/lib-storage/index/index-mail.c
@@ -95,8 +95,9 @@ int index_mail_cache_lookup_field(struct index_mail *mail, buffer_t *buf,
        if (_mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE_START_CACHING &&
            mail_cache_field_get_decision(_mail->box->cache, field_idx) ==
            MAIL_CACHE_DECISION_NO) {
+               bool rejected ATTR_UNUSED;
                mail_cache_decision_add(_mail->transaction->cache_view,
-                                       _mail->seq, field_idx);
+                                       _mail->seq, field_idx, &rejected);
        }
 
        return ret;

diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c
index 20cae26954cfc6526ea2ec1d2314246430e1c6fc..a8cdda52271e26204a6e8ae7495e55e5a382398a 100644 (file)
--- a/src/lib-storage/index/index-storage.c
+++ b/src/lib-storage/index/index-storage.c
@@ -289,6 +289,7 @@ int index_storage_mailbox_alloc_index(struct mailbox *box)
                .cache = {
                        .unaccessed_field_drop_secs = set->mail_cache_unaccessed_field_drop,
                        .record_max_size = set->mail_cache_record_max_size,
+                       .max_headers_count = set->mail_cache_max_headers_count,
                        .max_size = set->mail_cache_max_size,
                        .purge_min_size = set->mail_cache_purge_min_size,
                        .purge_delete_percentage = set->mail_cache_purge_delete_percentage,

diff --git a/src/lib-storage/mail-storage-settings.c b/src/lib-storage/mail-storage-settings.c
index e3973a4f7f203370506ed95e11f6c3f38a9ba3de..1e38e476dc1e1868177c34bb962cd3a49403485c 100644 (file)
--- a/src/lib-storage/mail-storage-settings.c
+++ b/src/lib-storage/mail-storage-settings.c
@@ -46,6 +46,7 @@ static const struct setting_define mail_storage_setting_defines[] = {
        DEF(STR, mail_server_admin),
        DEF(TIME_HIDDEN, mail_cache_unaccessed_field_drop),
        DEF(SIZE_HIDDEN, mail_cache_record_max_size),
+       DEF(UINT_HIDDEN, mail_cache_max_headers_count),
        DEF(SIZE_HIDDEN, mail_cache_max_size),
        DEF(UINT_HIDDEN, mail_cache_min_mail_count),
        DEF(SIZE_HIDDEN, mail_cache_purge_min_size),
@@ -103,6 +104,7 @@ const struct mail_storage_settings mail_storage_default_settings = {
        .mail_cache_min_mail_count = 0,
        .mail_cache_unaccessed_field_drop = 60*60*24*30,
        .mail_cache_record_max_size = 64 * 1024,
+       .mail_cache_max_headers_count = 100,
        .mail_cache_max_size = 1024 * 1024 * 1024,
        .mail_cache_purge_min_size = 32 * 1024,
        .mail_cache_purge_delete_percentage = 20,

diff --git a/src/lib-storage/mail-storage-settings.h b/src/lib-storage/mail-storage-settings.h
index 5a3cebbe165fc483cf85f3f0a3f5c9708aea8df2..eacc4095c3ee8c3d70aa30a6f49f8b4f0a29afa6 100644 (file)
--- a/src/lib-storage/mail-storage-settings.h
+++ b/src/lib-storage/mail-storage-settings.h
@@ -27,6 +27,7 @@ struct mail_storage_settings {
        unsigned int mail_cache_min_mail_count;
        unsigned int mail_cache_unaccessed_field_drop;
        uoff_t mail_cache_record_max_size;
+       unsigned int mail_cache_max_headers_count;
        uoff_t mail_cache_max_size;
        uoff_t mail_cache_purge_min_size;
        unsigned int mail_cache_purge_delete_percentage;