From: Marco Bettini <marco.bettini@open-xchange.com>
Date: Tue, 6 Dec 2022 11:29:48 +0000 (+0000)
Subject: global: Add mail_cache_max_headers_count functionality
X-Git-Tag: 2.4.0~3270
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4db593bd8787caa9485f1016a4070970997f2b6b;p=thirdparty%2Fdovecot%2Fcore.git

global: Add mail_cache_max_headers_count functionality
---

diff --git a/src/lib-index/mail-cache-decisions.c b/src/lib-index/mail-cache-decisions.c
index 4c26912bca..7d8baf7bf1 100644
--- a/src/lib-index/mail-cache-decisions.c
+++ b/src/lib-index/mail-cache-decisions.c
@@ -173,8 +173,44 @@ void mail_cache_decision_state_update(struct mail_cache_view *view,
 	}
 }
 
+static unsigned int mail_cache_count_alive_headers(struct mail_cache *cache)
+{
+	unsigned int count = 0;
+	for (unsigned int index = 0; index < cache->fields_count; ++index) {
+		if (cache->fields[index].field.type == MAIL_CACHE_FIELD_HEADER &&
+		    (cache->fields[index].field.decision &
+		     ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) !=
+						MAIL_CACHE_DECISION_NO)
+			++count;
+	}
+	return count;
+}
+
+bool mail_cache_headers_check_capped(struct mail_cache *cache)
+{
+	struct mail_index_cache_optimization_settings *set =
+		&cache->index->optimization_set.cache;
+
+	if (set->max_headers_count == 0) return FALSE;
+	if (cache->headers_capped) return TRUE;
+
+	unsigned int count = mail_cache_count_alive_headers(cache);
+	cache->headers_capped = count >= set->max_headers_count;
+	return cache->headers_capped;
+}
+
+static struct event_passthrough *
+mail_cache_decision_rejected_event(struct mail_cache *cache, unsigned int field,
+				   const char *reason)
+{
+	return event_create_passthrough(cache->event)->
+		set_name("mail_cache_decision_rejected")->
+		add_str("field", cache->fields[field].field.name)->
+		add_str("reason", reason);
+}
+
 void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq,
-			     unsigned int field)
+			     unsigned int field, bool *rejected_r)
 {
 	struct mail_cache *cache = view->cache;
 	struct mail_cache_field_private *priv;
@@ -182,6 +218,8 @@ void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq,
 
 	i_assert(field < cache->fields_count);
 
+	*rejected_r = FALSE;
+
 	if (view->no_decision_updates)
 		return;
 
@@ -194,8 +232,22 @@ void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq,
 	}
 
 	/* field used the first time */
-	if (priv->field.decision == MAIL_CACHE_DECISION_NO)
+	if (priv->field.decision == MAIL_CACHE_DECISION_NO) {
+		if (mail_cache_headers_check_capped(view->cache)) {
+			*rejected_r = TRUE;
+
+			const char *reason = "too_many_headers";
+			struct event_passthrough *e =
+				mail_cache_decision_rejected_event(
+					cache, field, reason);
+			e_debug(e->event(),
+				"Cache rejected header '%s': %s",
+				priv->field.name, reason);
+			return;
+		}
+
 		priv->field.decision = MAIL_CACHE_DECISION_TEMP;
+	}
 	priv->field.last_used = ioloop_time;
 	priv->decision_dirty = TRUE;
 	cache->field_header_write_pending = TRUE;
diff --git a/src/lib-index/mail-cache-fields.c b/src/lib-index/mail-cache-fields.c
index 429e0d234c..9a2f2c1af8 100644
--- a/src/lib-index/mail-cache-fields.c
+++ b/src/lib-index/mail-cache-fields.c
@@ -78,6 +78,12 @@ mail_cache_field_update(struct mail_cache *cache,
 	   internal fields and for mail_*cache_fields settings? */
 	initial_registering = cache->file_fields_count == 0;
 
+	if (newfield->decision == MAIL_CACHE_DECISION_NO) {
+		/* reset the cached flag, it will be set again anyway on first
+		   access if that is the case */
+		cache->headers_capped = FALSE;
+	}
+
 	orig = &cache->fields[newfield->idx];
 	if ((newfield->decision & MAIL_CACHE_DECISION_FORCED) != 0 ||
 	    ((orig->field.decision & MAIL_CACHE_DECISION_FORCED) == 0 &&
@@ -346,6 +352,10 @@ int mail_cache_header_fields_read(struct mail_cache *cache)
 		return -1;
 	}
 
+	/* reset the cached flag, it will be set again anyway on first access if
+	   that is the case */
+	cache->headers_capped = FALSE;
+
 	new_fields_count = field_hdr->fields_count;
 	if (new_fields_count != 0) {
 		cache->file_field_map =
diff --git a/src/lib-index/mail-cache-private.h b/src/lib-index/mail-cache-private.h
index c2fee17612..bb11a345bb 100644
--- a/src/lib-index/mail-cache-private.h
+++ b/src/lib-index/mail-cache-private.h
@@ -219,6 +219,8 @@ struct mail_cache {
 	   larger parts of it). This is used with MAIL_INDEX_OPEN_FLAG_SAVEONLY
 	   to avoid unnecessary cache reads. */
 	bool map_with_read:1;
+	/* Cache headers count has been capped */
+	bool headers_capped:1;
 };
 
 struct mail_cache_loop_track {
@@ -395,6 +397,8 @@ struct event_passthrough *
 mail_cache_decision_changed_event(struct mail_cache *cache, struct event *event,
 				  unsigned int field);
 
+bool mail_cache_headers_check_capped(struct mail_cache *cache);
+
 struct mail_cache_purge_drop_ctx {
 	struct mail_cache *cache;
 	time_t max_yes_downgrade_time;
diff --git a/src/lib-index/mail-cache-transaction.c b/src/lib-index/mail-cache-transaction.c
index 9d2c47a4d2..b766da1fa2 100644
--- a/src/lib-index/mail-cache-transaction.c
+++ b/src/lib-index/mail-cache-transaction.c
@@ -795,7 +795,12 @@ void mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
 	   read. */
 	mail_cache_transaction_refresh_decisions(ctx);
 
-	mail_cache_decision_add(ctx->view, seq, field_idx);
+	/* The decision to reject a field may come last minute during actual add,
+	   for example if the header count limit has been reached */
+	bool rejected;
+	mail_cache_decision_add(ctx->view, seq, field_idx, &rejected);
+	if (rejected)
+		return;
 
 	fixed_size = ctx->cache->fields[field_idx].field.field_size;
 	i_assert(fixed_size == UINT_MAX || fixed_size == data_size);
diff --git a/src/lib-index/mail-cache.h b/src/lib-index/mail-cache.h
index 09fde2941f..3a0c3db1b1 100644
--- a/src/lib-index/mail-cache.h
+++ b/src/lib-index/mail-cache.h
@@ -162,9 +162,10 @@ enum mail_cache_decision_type
 mail_cache_field_get_decision(struct mail_cache *cache, unsigned int field_idx);
 /* Notify the decision handling code when field is committed to cache.
    If this is the first time the field is added to cache, its caching decision
-   is updated to TEMP. */
+   is updated to TEMP. Sets rejected to true if header count limit has been
+   reached and the field has NOT been switched to TEMP */
 void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq,
-			     unsigned int field);
+			     unsigned int field, bool *rejected_r);
 
 /* Set data_r and size_r to point to wanted field in cache file.
    Returns 1 if field was found, 0 if not, -1 if error. */
diff --git a/src/lib-index/mail-index.c b/src/lib-index/mail-index.c
index c0e6a4fe06..01bb93348d 100644
--- a/src/lib-index/mail-index.c
+++ b/src/lib-index/mail-index.c
@@ -226,6 +226,8 @@ void mail_index_set_optimization_settings(struct mail_index *index,
 			set->cache.purge_header_continue_count;
 	if (set->cache.record_max_size != 0)
 		dest->cache.record_max_size = set->cache.record_max_size;
+
+	dest->cache.max_headers_count = set->cache.max_headers_count;
 }
 
 void mail_index_set_ext_init_data(struct mail_index *index, uint32_t ext_id,
diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h
index 6d768e050b..c401fc679e 100644
--- a/src/lib-index/mail-index.h
+++ b/src/lib-index/mail-index.h
@@ -331,6 +331,9 @@ struct mail_index_cache_optimization_settings {
 	/* If cache record becomes larger than this, don't add it. */
 	unsigned int record_max_size;
 
+	/* Maximum number of headers to cache */
+	unsigned int max_headers_count;
+
 	/* Maximum size for the cache file. Internally the limit is 1 GB. */
 	uoff_t max_size;
 	/* Never purge the file if it's smaller than this */
diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c
index 12079a1751..5a2d45398e 100644
--- a/src/lib-storage/index/index-mail.c
+++ b/src/lib-storage/index/index-mail.c
@@ -95,8 +95,9 @@ int index_mail_cache_lookup_field(struct index_mail *mail, buffer_t *buf,
 	if (_mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE_START_CACHING &&
 	    mail_cache_field_get_decision(_mail->box->cache, field_idx) ==
 	    MAIL_CACHE_DECISION_NO) {
+		bool rejected ATTR_UNUSED;
 		mail_cache_decision_add(_mail->transaction->cache_view,
-					_mail->seq, field_idx);
+					_mail->seq, field_idx, &rejected);
 	}
 
 	return ret;
diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c
index 20cae26954..a8cdda5227 100644
--- a/src/lib-storage/index/index-storage.c
+++ b/src/lib-storage/index/index-storage.c
@@ -289,6 +289,7 @@ int index_storage_mailbox_alloc_index(struct mailbox *box)
 		.cache = {
 			.unaccessed_field_drop_secs = set->mail_cache_unaccessed_field_drop,
 			.record_max_size = set->mail_cache_record_max_size,
+			.max_headers_count = set->mail_cache_max_headers_count,
 			.max_size = set->mail_cache_max_size,
 			.purge_min_size = set->mail_cache_purge_min_size,
 			.purge_delete_percentage = set->mail_cache_purge_delete_percentage,
diff --git a/src/lib-storage/mail-storage-settings.c b/src/lib-storage/mail-storage-settings.c
index e3973a4f7f..1e38e476dc 100644
--- a/src/lib-storage/mail-storage-settings.c
+++ b/src/lib-storage/mail-storage-settings.c
@@ -46,6 +46,7 @@ static const struct setting_define mail_storage_setting_defines[] = {
 	DEF(STR, mail_server_admin),
 	DEF(TIME_HIDDEN, mail_cache_unaccessed_field_drop),
 	DEF(SIZE_HIDDEN, mail_cache_record_max_size),
+	DEF(UINT_HIDDEN, mail_cache_max_headers_count),
 	DEF(SIZE_HIDDEN, mail_cache_max_size),
 	DEF(UINT_HIDDEN, mail_cache_min_mail_count),
 	DEF(SIZE_HIDDEN, mail_cache_purge_min_size),
@@ -103,6 +104,7 @@ const struct mail_storage_settings mail_storage_default_settings = {
 	.mail_cache_min_mail_count = 0,
 	.mail_cache_unaccessed_field_drop = 60*60*24*30,
 	.mail_cache_record_max_size = 64 * 1024,
+	.mail_cache_max_headers_count = 100,
 	.mail_cache_max_size = 1024 * 1024 * 1024,
 	.mail_cache_purge_min_size = 32 * 1024,
 	.mail_cache_purge_delete_percentage = 20,
diff --git a/src/lib-storage/mail-storage-settings.h b/src/lib-storage/mail-storage-settings.h
index 5a3cebbe16..eacc4095c3 100644
--- a/src/lib-storage/mail-storage-settings.h
+++ b/src/lib-storage/mail-storage-settings.h
@@ -27,6 +27,7 @@ struct mail_storage_settings {
 	unsigned int mail_cache_min_mail_count;
 	unsigned int mail_cache_unaccessed_field_drop;
 	uoff_t mail_cache_record_max_size;
+	unsigned int mail_cache_max_headers_count;
 	uoff_t mail_cache_max_size;
 	uoff_t mail_cache_purge_min_size;
 	unsigned int mail_cache_purge_delete_percentage;