lib-storage: Add mail_sort_max_read_count setting.

This controls how many slow mail accesses sorting can perform before it
fails:

a NO [LIMIT] Requested sort would have taken too long

The SORT reply is still returned, but it's likely not correct.

diff --git a/src/lib-storage/index/index-sort-private.h b/src/lib-storage/index/index-sort-private.h
index 54907f0fd51ae743099091ced1ce2798a456ee03..3dd07720daca058c4e3087c136d8edd37ad96db2 100644 (file)
--- a/src/lib-storage/index/index-sort-private.h
+++ b/src/lib-storage/index/index-sort-private.h
@@ -7,6 +7,7 @@ struct mail_search_sort_program {
        struct mailbox_transaction_context *t;
        enum mail_sort_type sort_program[MAX_SORT_PROGRAM_SIZE];
        struct mail *temp_mail;
+       unsigned int slow_mails_left;
 
        void (*sort_list_add)(struct mail_search_sort_program *program,
                              struct mail *mail);
@@ -20,7 +21,7 @@ struct mail_search_sort_program {
 };
 
 /* Returns 1 on success, 0 if mail is already expunged, -1 on other errors. */
-int index_sort_header_get(struct mail *mail, uint32_t seq,
+int index_sort_header_get(struct mail_search_sort_program *program, uint32_t seq,
                          enum mail_sort_type sort_type, string_t *dest);
 int index_sort_node_cmp_type(struct mail_search_sort_program *program,
                             const enum mail_sort_type *sort_program,

diff --git a/src/lib-storage/index/index-sort-string.c b/src/lib-storage/index/index-sort-string.c
index 814828fcde5b52713820095b161a25c40dc787c4..362e2121ad9e08ebf2fd9560591fefcdfa746321 100644 (file)
--- a/src/lib-storage/index/index-sort-string.c
+++ b/src/lib-storage/index/index-sort-string.c
@@ -254,7 +254,6 @@ static int sort_node_zero_string_cmp(const struct mail_sort_node *n1,
 
 static void index_sort_zeroes(struct sort_string_context *ctx)
 {
-       struct mail *mail = ctx->program->temp_mail;
        enum mail_sort_type sort_type = ctx->program->sort_program[0];
        string_t *str;
        pool_t pool;
@@ -273,7 +272,7 @@ static void index_sort_zeroes(struct sort_string_context *ctx)
                i_assert(nodes[i].seq <= ctx->last_seq);
 
                T_BEGIN {
-                       if (index_sort_header_get(mail, nodes[i].seq,
+                       if (index_sort_header_get(ctx->program, nodes[i].seq,
                                                  sort_type, str) < 0) {
                                nodes[i].no_update = TRUE;
                                ctx->failed = TRUE;
@@ -294,7 +293,6 @@ static bool
 index_sort_get_expunged_string(struct sort_string_context *ctx, uint32_t idx,
                               string_t *str, const char **result_r)
 {
-       struct mail *mail = ctx->program->temp_mail;
        enum mail_sort_type sort_type = ctx->program->sort_program[0];
        const struct mail_sort_node *nodes;
        const char *result = NULL;
@@ -329,7 +327,7 @@ index_sort_get_expunged_string(struct sort_string_context *ctx, uint32_t idx,
                        result = ctx->sort_strings[nodes[i].seq];
                        break;
                }
-               if (index_sort_header_get(mail, nodes[i].seq,
+               if (index_sort_header_get(ctx->program, nodes[i].seq,
                                          sort_type, str) > 0) {
                        result = str_len(str) == 0 ? "" :
                                p_strdup(ctx->sort_string_pool, str_c(str));
@@ -353,7 +351,6 @@ static int
 index_sort_get_string(struct sort_string_context *ctx,
                      uint32_t idx, uint32_t seq, const char **str_r)
 {
-       struct mail *mail = ctx->program->temp_mail;
        int ret = 1;
 
        if (ctx->sort_strings[seq] == NULL) T_BEGIN {
@@ -361,7 +358,7 @@ index_sort_get_string(struct sort_string_context *ctx,
                const char *result;
 
                str = t_str_new(256);
-               ret = index_sort_header_get(mail, seq,
+               ret = index_sort_header_get(ctx->program, seq,
                                            ctx->program->sort_program[0], str);
                if (ret < 0)
                        ctx->failed = TRUE;

diff --git a/src/lib-storage/index/index-sort.c b/src/lib-storage/index/index-sort.c
index 66d139abe24f9af7f57ba98e873ba11af299c44d..452395a6185efb7d8b59999f0c10ab05965085b3 100644 (file)
--- a/src/lib-storage/index/index-sort.c
+++ b/src/lib-storage/index/index-sort.c
@@ -40,8 +40,34 @@ static void
 index_sort_program_set_mail_failed(struct mail_search_sort_program *program,
                                   struct mail *mail)
 {
-       if (mailbox_get_last_mail_error(mail->box) != MAIL_ERROR_EXPUNGED)
+       switch (mailbox_get_last_mail_error(mail->box)) {
+       case MAIL_ERROR_EXPUNGED:
+               break;
+       case MAIL_ERROR_NOTPOSSIBLE:
+               /* just change the error message */
+               mail_storage_set_error(program->t->box->storage, MAIL_ERROR_LIMIT,
+                       "Requested sort would have taken too long.");
+               /* fall through */
+       default:
                program->failed = TRUE;
+               break;
+       }
+}
+
+static time_t
+index_sort_program_set_date_failed(struct mail_search_sort_program *program,
+                                  struct mail *mail)
+{
+       index_sort_program_set_mail_failed(program, mail);
+
+       if (mailbox_get_last_mail_error(mail->box) == MAIL_ERROR_LIMIT) {
+               /* limit reached - sort the rest of the mails at the end of
+                  the list by their UIDs */
+               return LONG_MAX;
+       } else {
+               /* expunged / some other error - sort in the beginning */
+               return 0;
+       }
 }
 
 static void
@@ -53,10 +79,8 @@ index_sort_list_add_arrival(struct mail_search_sort_program *program,
 
        node = array_append_space(nodes);
        node->seq = mail->seq;
-       if (mail_get_received_date(mail, &node->date) < 0) {
-               index_sort_program_set_mail_failed(program, mail);
-               node->date = 0;
-       }
+       if (mail_get_received_date(mail, &node->date) < 0)
+               node->date = index_sort_program_set_date_failed(program, mail);
 }
 
 static void
@@ -69,14 +93,11 @@ index_sort_list_add_date(struct mail_search_sort_program *program,
 
        node = array_append_space(nodes);
        node->seq = mail->seq;
-       if (mail_get_date(mail, &node->date, &tz) < 0) {
-               index_sort_program_set_mail_failed(program, mail);
-               node->date = 0;
-       } else if (node->date == 0) {
-               if (mail_get_received_date(mail, &node->date) < 0) {
-                       index_sort_program_set_mail_failed(program, mail);
-                       node->date = 0;
-               }
+       if (mail_get_date(mail, &node->date, &tz) < 0)
+               node->date = index_sort_program_set_date_failed(program, mail);
+       else if (node->date == 0) {
+               if (mail_get_received_date(mail, &node->date) < 0)
+                       node->date = index_sort_program_set_date_failed(program, mail);
        }
 }
 
@@ -148,11 +169,23 @@ index_sort_list_add_relevancy(struct mail_search_sort_program *program,
 void index_sort_list_add(struct mail_search_sort_program *program,
                         struct mail *mail)
 {
+       enum mail_lookup_abort orig_abort = mail->lookup_abort;
+       bool prev_slow = mail->mail_stream_opened ||
+               mail->mail_metadata_accessed;
+
        i_assert(mail->transaction == program->t);
 
+       if (program->slow_mails_left == 0)
+               mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
        T_BEGIN {
                program->sort_list_add(program, mail);
        } T_END;
+       if (!prev_slow && (mail->mail_stream_opened ||
+                          mail->mail_metadata_accessed)) {
+               i_assert(program->slow_mails_left > 0);
+               program->slow_mails_left--;
+       }
+       mail->lookup_abort = orig_abort;
 }
 
 static int sort_node_date_cmp(const struct mail_sort_node_date *n1,
@@ -276,6 +309,11 @@ index_sort_program_init(struct mailbox_transaction_context *t,
        program->t = t;
        program->temp_mail = mail_alloc(t, 0, NULL);
 
+       program->slow_mails_left =
+               program->t->box->storage->set->mail_sort_max_read_count;
+       if (program->slow_mails_left == 0)
+               program->slow_mails_left = UINT_MAX;
+
        for (i = 0; i < MAX_SORT_PROGRAM_SIZE; i++) {
                program->sort_program[i] = sort_program[i];
                if (sort_program[i] == MAIL_SORT_END)
@@ -425,14 +463,30 @@ get_display_name(struct mail *mail, const char *header, const char **name_r)
        return 0;
 }
 
-int index_sort_header_get(struct mail *mail, uint32_t seq,
+static void
+index_sort_set_seq(struct mail_search_sort_program *program,
+                  struct mail *mail, uint32_t seq)
+{
+       if ((mail->mail_stream_opened || mail->mail_metadata_accessed) &&
+           program->slow_mails_left > 0)
+               program->slow_mails_left--;
+       mail_set_seq(mail, seq);
+       if (program->slow_mails_left == 0) {
+               /* too many slow lookups - just return the rest of the results
+                  in whatever order. */
+               mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+       }
+}
+
+int index_sort_header_get(struct mail_search_sort_program *program, uint32_t seq,
                          enum mail_sort_type sort_type, string_t *dest)
 {
+       struct mail *mail = program->temp_mail;
        const char *str;
        int ret;
        bool reply_or_fw;
 
-       mail_set_seq(mail, seq);
+       index_sort_set_seq(program, mail, seq);
        str_truncate(dest, 0);
 
        switch (sort_type & MAIL_SORT_MASK) {
@@ -500,64 +554,54 @@ int index_sort_node_cmp_type(struct mail_search_sort_program *program,
 
                        str1 = t_str_new(256);
                        str2 = t_str_new(256);
-                       if (index_sort_header_get(mail, seq1, sort_type, str1) < 0)
+                       if (index_sort_header_get(program, seq1, sort_type, str1) < 0)
                                index_sort_program_set_mail_failed(program, mail);
-                       if (index_sort_header_get(mail, seq2, sort_type, str2) < 0)
+                       if (index_sort_header_get(program, seq2, sort_type, str2) < 0)
                                index_sort_program_set_mail_failed(program, mail);
 
                        ret = strcmp(str_c(str1), str_c(str2));
                } T_END;
                break;
        case MAIL_SORT_ARRIVAL:
-               mail_set_seq(mail, seq1);
-               if (mail_get_received_date(mail, &time1) < 0) {
-                       index_sort_program_set_mail_failed(program, mail);
-                       time1 = 0;
-               }
+               index_sort_set_seq(program, mail, seq1);
+               if (mail_get_received_date(mail, &time1) < 0)
+                       time1 = index_sort_program_set_date_failed(program, mail);
 
-               mail_set_seq(mail, seq2);
-               if (mail_get_received_date(mail, &time2) < 0) {
-                       index_sort_program_set_mail_failed(program, mail);
-                       time2 = 0;
-               }
+               index_sort_set_seq(program, mail, seq2);
+               if (mail_get_received_date(mail, &time2) < 0)
+                       time2 = index_sort_program_set_date_failed(program, mail);
 
                ret = time1 < time2 ? -1 :
                        (time1 > time2 ? 1 : 0);
                break;
        case MAIL_SORT_DATE:
-               mail_set_seq(mail, seq1);
-               if (mail_get_date(mail, &time1, &tz) < 0) {
-                       index_sort_program_set_mail_failed(program, mail);
-                       time1 = 0;
-               } else if (time1 == 0) {
-                       if (mail_get_received_date(mail, &time1) < 0) {
-                               index_sort_program_set_mail_failed(program, mail);
-                               time1 = 0;
-                       }
+               index_sort_set_seq(program, mail, seq1);
+               if (mail_get_date(mail, &time1, &tz) < 0)
+                       time1 = index_sort_program_set_date_failed(program, mail);
+               else if (time1 == 0) {
+                       if (mail_get_received_date(mail, &time1) < 0)
+                               time1 = index_sort_program_set_date_failed(program, mail);
                }
 
-               mail_set_seq(mail, seq2);
-               if (mail_get_date(mail, &time2, &tz) < 0) {
-                       index_sort_program_set_mail_failed(program, mail);
-                       time2 = 0;
-               } else if (time2 == 0) {
-                       if (mail_get_received_date(mail, &time2) < 0) {
-                               index_sort_program_set_mail_failed(program, mail);
-                               time2 = 0;
-                       }
+               index_sort_set_seq(program, mail, seq2);
+               if (mail_get_date(mail, &time2, &tz) < 0)
+                       time2 = index_sort_program_set_date_failed(program, mail);
+               else if (time2 == 0) {
+                       if (mail_get_received_date(mail, &time2) < 0)
+                               time2 = index_sort_program_set_date_failed(program, mail);
                }
 
                ret = time1 < time2 ? -1 :
                        (time1 > time2 ? 1 : 0);
                break;
        case MAIL_SORT_SIZE:
-               mail_set_seq(mail, seq1);
+               index_sort_set_seq(program, mail, seq1);
                if (mail_get_virtual_size(mail, &size1) < 0) {
                        index_sort_program_set_mail_failed(program, mail);
                        size1 = 0;
                }
 
-               mail_set_seq(mail, seq2);
+               index_sort_set_seq(program, mail, seq2);
                if (mail_get_virtual_size(mail, &size2) < 0) {
                        index_sort_program_set_mail_failed(program, mail);
                        size2 = 0;
@@ -567,10 +611,10 @@ int index_sort_node_cmp_type(struct mail_search_sort_program *program,
                        (size1 > size2 ? 1 : 0);
                break;
        case MAIL_SORT_RELEVANCY:
-               mail_set_seq(mail, seq1);
+               index_sort_set_seq(program, mail, seq1);
                if (index_sort_get_relevancy(mail, &float1) < 0)
                        index_sort_program_set_mail_failed(program, mail);
-               mail_set_seq(mail, seq2);
+               index_sort_set_seq(program, mail, seq2);
                if (index_sort_get_relevancy(mail, &float2) < 0)
                        index_sort_program_set_mail_failed(program, mail);
 
@@ -582,10 +626,10 @@ int index_sort_node_cmp_type(struct mail_search_sort_program *program,
        case MAIL_SORT_POP3_ORDER:
                /* 32bit numbers would be enough, but since there is already
                   existing code for uoff_t in sizes, just use them. */
-               mail_set_seq(mail, seq1);
+               index_sort_set_seq(program, mail, seq1);
                if (index_sort_get_pop3_order(mail, &size1) < 0)
                        index_sort_program_set_mail_failed(program, mail);
-               mail_set_seq(mail, seq2);
+               index_sort_set_seq(program, mail, seq2);
                if (index_sort_get_pop3_order(mail, &size2) < 0)
                        index_sort_program_set_mail_failed(program, mail);
 

diff --git a/src/lib-storage/mail-storage-settings.c b/src/lib-storage/mail-storage-settings.c
index 0990bc8f42f31b840dd6036e7e30ac54b16ca91c..dea6a0dbd14da925af88e0416cd9a6a070887050 100644 (file)
--- a/src/lib-storage/mail-storage-settings.c
+++ b/src/lib-storage/mail-storage-settings.c
@@ -43,6 +43,7 @@ static const struct setting_define mail_storage_setting_defines[] = {
        DEF(SET_TIME, mail_max_lock_timeout),
        DEF(SET_TIME, mail_temp_scan_interval),
        DEF(SET_UINT, mail_vsize_bg_after_count),
+       DEF(SET_UINT, mail_sort_max_read_count),
        DEF(SET_BOOL, mail_save_crlf),
        DEF(SET_ENUM, mail_fsync),
        DEF(SET_BOOL, mmap_disable),
@@ -84,6 +85,7 @@ const struct mail_storage_settings mail_storage_default_settings = {
        .mail_max_lock_timeout = 0,
        .mail_temp_scan_interval = 7*24*60*60,
        .mail_vsize_bg_after_count = 0,
+       .mail_sort_max_read_count = 0,
        .mail_save_crlf = FALSE,
        .mail_fsync = "optimized:never:always",
        .mmap_disable = FALSE,

diff --git a/src/lib-storage/mail-storage-settings.h b/src/lib-storage/mail-storage-settings.h
index a292e222d36c9b26a5db120bc93153e3c23c7495..409669d26d91c1e35ee4e5277235b68635011227 100644 (file)
--- a/src/lib-storage/mail-storage-settings.h
+++ b/src/lib-storage/mail-storage-settings.h
@@ -29,6 +29,7 @@ struct mail_storage_settings {
        unsigned int mail_max_lock_timeout;
        unsigned int mail_temp_scan_interval;
        unsigned int mail_vsize_bg_after_count;
+       unsigned int mail_sort_max_read_count;
        bool mail_save_crlf;
        const char *mail_fsync;
        bool mmap_disable;