From: Marco Bettini Date: Fri, 24 Jan 2025 16:06:17 +0000 (+0000) Subject: fts-flatcurve: Fix maybe queries X-Git-Tag: 2.4.1~174 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=bb648e3b979a427ee8c6e10988cd7b6612f3d273;p=thirdparty%2Fdovecot%2Fcore.git fts-flatcurve: Fix maybe queries --- diff --git a/src/plugins/fts-flatcurve/fts-backend-flatcurve-xapian.cc b/src/plugins/fts-flatcurve/fts-backend-flatcurve-xapian.cc index e24a81821d..0873b3515d 100644 --- a/src/plugins/fts-flatcurve/fts-backend-flatcurve-xapian.cc +++ b/src/plugins/fts-flatcurve/fts-backend-flatcurve-xapian.cc @@ -3,6 +3,7 @@ extern "C" { #include "lib.h" +#include "array.h" #include "file-create-locked.h" #include "hash.h" #include "message-header-parser.h" @@ -159,10 +160,19 @@ struct flatcurve_xapian { bool deinit:1; }; -struct flatcurve_fts_query_xapian { +struct flatcurve_fts_query_xapian_maybe { Xapian::Query *query; }; + struct flatcurve_fts_query_xapian { + Xapian::Query *query; + ARRAY(struct flatcurve_fts_query_xapian_maybe) maybe_queries; + + bool and_search:1; + bool maybe:1; + bool start:1; + }; + struct flatcurve_xapian_db_iter { struct flatcurve_fts_backend *backend; DIR *dirp; @@ -200,6 +210,8 @@ struct fts_flatcurve_xapian_query_iter { Xapian::Database *db; Xapian::Enquire *enquire; Xapian::MSetIterator mset_iter; + int curr_query; + bool next_query:1; }; static int @@ -1964,12 +1976,14 @@ fts_flatcurve_build_query_arg_term(struct flatcurve_fts_query *query, const char *term) { const char *hdr; + bool maybe_or = FALSE; + struct flatcurve_fts_query_xapian_maybe *mquery; Xapian::Query::op op = Xapian::Query::OP_INVALID; Xapian::Query *oldq, q; struct flatcurve_fts_query_xapian *x = query->xapian; - if (x->query != NULL) { - if ((query->flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0) { + if (x->start) { + if (x->and_search) { op = Xapian::Query::OP_AND; str_append(query->qtext, " AND "); } else { @@ -1977,6 +1991,7 @@ fts_flatcurve_build_query_arg_term(struct flatcurve_fts_query *query, str_append(query->qtext, " OR "); } } + x->start = TRUE; if (arg->match_not) str_append(query->qtext, "NOT "); @@ -2028,7 +2043,10 @@ fts_flatcurve_build_query_arg_term(struct flatcurve_fts_query *query, * appears in the general pool of header * terms for the message, not to a specific * header, so this is only a maybe match. */ - query->maybe = TRUE; + if (x->and_search) + x->maybe = TRUE; + else + maybe_or = TRUE; } } else { hdr = t_str_lcase(arg->hdr_field_name); @@ -2046,9 +2064,17 @@ fts_flatcurve_build_query_arg_term(struct flatcurve_fts_query *query, q = Xapian::Query(Xapian::Query::OP_AND_NOT, Xapian::Query::MatchAll, q); - if (x->query == NULL) + if (maybe_or) { + /* Maybe searches are not added to the "master search" query if this + * is an OR search; they will be run independently. Matches will be + * placed in the maybe results array. */ + if (!array_is_created(&x->maybe_queries)) + p_array_init(&x->maybe_queries, query->pool, 4); + mquery = array_append_space(&x->maybe_queries); + mquery->query = new Xapian::Query(std_move(q)); + } else if (x->query == NULL) { x->query = new Xapian::Query(std_move(q)); - else { + } else { oldq = x->query; x->query = new Xapian::Query(op, *(x->query), q); delete(oldq); @@ -2125,6 +2151,7 @@ void fts_flatcurve_xapian_build_query(struct flatcurve_fts_query *query) struct mail_search_arg *args; query->xapian = p_new(query->pool, struct flatcurve_fts_query_xapian, 1); + query->xapian->and_search = ((query->flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0); for (args = query->args; args != NULL ; args = args->next) fts_flatcurve_build_query_arg(query, args); } @@ -2134,6 +2161,8 @@ fts_flatcurve_xapian_query_iter_init(struct flatcurve_fts_query *query) { struct fts_flatcurve_xapian_query_iter *iter; iter = new fts_flatcurve_xapian_query_iter(); + iter->curr_query = -1; + iter->next_query = TRUE; iter->query = query; iter->result = p_new(query->pool, struct fts_flatcurve_xapian_query_result, 1); @@ -2151,22 +2180,45 @@ fts_flatcurve_xapian_query_iter_next(struct fts_flatcurve_xapian_query_iter *ite return FALSE; Xapian::MSet m; - if (iter->enquire == NULL) { - if (iter->query->xapian->query == NULL) - return FALSE; + Xapian::Query *q = NULL; + if (iter->next_query) { + iter->next_query = FALSE; - const char *error; - int ret = fts_flatcurve_xapian_read_db( - iter->query->backend, opts, &iter->db, &error); - if (ret < 0) - iter->error = i_strdup(error); - if (ret <= 0) + if (iter->curr_query == -1) { + if (iter->query->xapian->query == NULL) + ++iter->curr_query; + else + q = iter->query->xapian->query; + } + + /* Maybe queries. */ + if ((iter->curr_query >= 0) && + (array_not_empty(&iter->query->xapian->maybe_queries)) && + (array_count(&iter->query->xapian->maybe_queries) > iter->curr_query)) { + const struct flatcurve_fts_query_xapian_maybe *mquery; + mquery = array_idx(&iter->query->xapian->maybe_queries, + iter->curr_query); + q = mquery->query; + } + + if (q == NULL) return FALSE; - iter->enquire = new Xapian::Enquire(*iter->db); - iter->enquire->set_docid_order( - Xapian::Enquire::DONT_CARE); - iter->enquire->set_query(*iter->query->xapian->query); + if (iter->db == NULL) { + const char *error; + int ret = fts_flatcurve_xapian_read_db( + iter->query->backend, opts, &iter->db, &error); + if (ret < 0) + iter->error = i_strdup(error); + if (ret <= 0) + return FALSE; + } + + if (iter->enquire == NULL) { + iter->enquire = new Xapian::Enquire(*iter->db); + iter->enquire->set_docid_order(Xapian::Enquire::DONT_CARE); + } + iter->enquire->set_query(*q); try { m = iter->enquire->get_mset(0, iter->db->get_doccount()); @@ -2186,9 +2238,13 @@ fts_flatcurve_xapian_query_iter_next(struct fts_flatcurve_xapian_query_iter *ite iter->mset_iter = m.begin(); } - if (iter->mset_iter == m.end()) - return FALSE; + if (iter->mset_iter == m.end()) { + ++iter->curr_query; + iter->next_query = TRUE; + return fts_flatcurve_xapian_query_iter_next(iter, result_r); + } + iter->result->maybe = (iter->curr_query >= 0); iter->result->score = iter->mset_iter.get_weight(); /* MSet docid can be an "interleaved" docid generated by * Xapian::Database when handling multiple DBs at once. Instead, we @@ -2234,7 +2290,10 @@ int fts_flatcurve_xapian_run_query(struct flatcurve_fts_query *query, iter = fts_flatcurve_xapian_query_iter_init(query); while (fts_flatcurve_xapian_query_iter_next(iter, &result)) { - seq_range_array_add(&r->uids, result->uid); + if (result->maybe || query->xapian->maybe) + seq_range_array_add(&r->maybe_uids, result->uid); + else + seq_range_array_add(&r->uids, result->uid); score = array_append_space(&r->scores); score->score = (float)result->score; score->uid = result->uid; @@ -2245,6 +2304,14 @@ int fts_flatcurve_xapian_run_query(struct flatcurve_fts_query *query, void fts_flatcurve_xapian_destroy_query(struct flatcurve_fts_query *query) { delete(query->xapian->query); + + if (array_is_created(&query->xapian->maybe_queries)) { + struct flatcurve_fts_query_xapian_maybe *mquery; + array_foreach_modifiable(&query->xapian->maybe_queries, mquery) { + delete(mquery->query); + } + array_free(&query->xapian->maybe_queries); + } } const char *fts_flatcurve_xapian_library_version() diff --git a/src/plugins/fts-flatcurve/fts-backend-flatcurve-xapian.h b/src/plugins/fts-flatcurve/fts-backend-flatcurve-xapian.h index 0e1d5d08fd..aa6551acd8 100644 --- a/src/plugins/fts-flatcurve/fts-backend-flatcurve-xapian.h +++ b/src/plugins/fts-flatcurve/fts-backend-flatcurve-xapian.h @@ -7,6 +7,8 @@ struct fts_flatcurve_xapian_query_result { double score; uint32_t uid; + + bool maybe:1; }; struct fts_flatcurve_xapian_db_check { diff --git a/src/plugins/fts-flatcurve/fts-backend-flatcurve.c b/src/plugins/fts-flatcurve/fts-backend-flatcurve.c index 9be6f2505d..7e663e5ee5 100644 --- a/src/plugins/fts-flatcurve/fts-backend-flatcurve.c +++ b/src/plugins/fts-flatcurve/fts-backend-flatcurve.c @@ -590,6 +590,7 @@ fts_backend_flatcurve_lookup_multi(struct fts_backend *_backend, r->box = boxes[i]; fresult = p_new(result->pool, struct flatcurve_fts_result, 1); + p_array_init(&fresult->maybe_uids, result->pool, 32); p_array_init(&fresult->scores, result->pool, 32); p_array_init(&fresult->uids, result->pool, 32); @@ -603,10 +604,8 @@ fts_backend_flatcurve_lookup_multi(struct fts_backend *_backend, break; } - if (query->maybe) - r->maybe_uids = fresult->uids; - else - r->definite_uids = fresult->uids; + r->definite_uids = fresult->uids; + r->maybe_uids = fresult->maybe_uids; r->scores = fresult->scores; if (str_len(query->qtext) == 0) { @@ -615,17 +614,25 @@ fts_backend_flatcurve_lookup_multi(struct fts_backend *_backend, } T_BEGIN { - const char *u = fts_backend_flatcurve_seq_range_string(&fresult->uids); + const char *m_debug = "", *u_debug = ""; + + if (array_not_empty(&fresult->maybe_uids)) + m_debug = fts_backend_flatcurve_seq_range_string( + &fresult->maybe_uids); + if (array_not_empty(&fresult->uids)) + u_debug = fts_backend_flatcurve_seq_range_string( + &fresult->uids); + e_debug(event_create_passthrough(backend->event)-> set_name("fts_flatcurve_query")-> - add_int("count", array_count(&fresult->uids))-> + add_int("count", seq_range_count(&fresult->uids))-> add_str("mailbox", r->box->vname)-> - add_str("maybe", query->maybe ? "yes" : "no")-> + add_str("maybe_uids", m_debug)-> add_str("query", str_c(query->qtext))-> - add_str("uids", u)->event(), "Query (%s) " - "%smatches=%d uids=%s", str_c(query->qtext), - query->maybe ? "maybe_" : "", - array_count(&fresult->uids), u); + add_str("uids", u_debug)->event(), "Query (%s) " + "matches=%d uids=%s maybe_matches=%d maybe_uids=%s", + str_c(query->qtext), seq_range_count(&fresult->uids), + u_debug, seq_range_count(&fresult->maybe_uids), m_debug); } T_END; } diff --git a/src/plugins/fts-flatcurve/fts-backend-flatcurve.h b/src/plugins/fts-flatcurve/fts-backend-flatcurve.h index deb9697531..c2e1c6360e 100644 --- a/src/plugins/fts-flatcurve/fts-backend-flatcurve.h +++ b/src/plugins/fts-flatcurve/fts-backend-flatcurve.h @@ -46,12 +46,11 @@ struct flatcurve_fts_query { struct flatcurve_fts_query_xapian *xapian; pool_t pool; - - bool maybe:1; }; struct flatcurve_fts_result { ARRAY_TYPE(fts_score_map) scores; + ARRAY_TYPE(seq_range) maybe_uids; ARRAY_TYPE(seq_range) uids; };