From f47f2aee21ec4288e6acbec998c033f295706f93 Mon Sep 17 00:00:00 2001 From: Marco Bettini Date: Wed, 17 May 2023 12:33:16 +0000 Subject: [PATCH] indexer: Provide progress updates in messages counts rather than just percentage (client side) --- src/indexer/indexer-client.c | 13 +++++++---- src/indexer/indexer-client.h | 4 +++- src/indexer/indexer-queue.c | 19 ++++++++-------- src/indexer/indexer-queue.h | 7 +++--- src/indexer/indexer.c | 16 ++++++-------- src/indexer/indexer.h | 4 ++-- src/indexer/test-indexer-queue.c | 21 +++++++++--------- src/indexer/worker-connection.c | 29 ++++++++++--------------- src/plugins/fts/fts-indexer.c | 37 ++++++++++++++++++++++++-------- 9 files changed, 85 insertions(+), 65 deletions(-) diff --git a/src/indexer/indexer-client.c b/src/indexer/indexer-client.c index 8bc9080beb..c9ed95c9da 100644 --- a/src/indexer/indexer-client.c +++ b/src/indexer/indexer-client.c @@ -253,15 +253,20 @@ indexer_client_input_args(struct connection *conn, const char *const *args) return 1; } -void indexer_client_status_callback(int percentage, void *context) +void indexer_client_status_callback(const struct indexer_status *status, + void *context) { struct indexer_client_request *ctx = context; if (ctx->client->conn.output != NULL) T_BEGIN { - o_stream_nsend_str(ctx->client->conn.output, - t_strdup_printf("%u\t%d\n", ctx->tag, percentage)); + const char *update = t_strdup_printf( + "%u\t%d\t%u\t%u\n", ctx->tag, + status->state, status->progress, status->total); + o_stream_nsend_str(ctx->client->conn.output, update); + } T_END; - if (percentage < 0 || percentage == 100) { + + if (status->state != INDEXER_STATE_PROCESSING) { indexer_client_unref(ctx->client); i_free(ctx); } diff --git a/src/indexer/indexer-client.h b/src/indexer/indexer-client.h index 609c67805a..99efa413f3 100644 --- a/src/indexer/indexer-client.h +++ b/src/indexer/indexer-client.h @@ -1,11 +1,13 @@ #ifndef INDEXER_CLIENT_H #define INDEXER_CLIENT_H +struct indexer_status; struct indexer_queue; void indexer_client_create(struct master_service_connection *conn, struct indexer_queue *queue); -void indexer_client_status_callback(int percentage, void *context); +void indexer_client_status_callback(const struct indexer_status *status, + void *context); unsigned int indexer_clients_get_count(void); void indexer_clients_destroy_all(void); diff --git a/src/indexer/indexer-queue.c b/src/indexer/indexer-queue.c index 5df538bc78..ee29fa3351 100644 --- a/src/indexer/indexer-queue.c +++ b/src/indexer/indexer-queue.c @@ -199,24 +199,23 @@ void indexer_queue_request_remove(struct indexer_queue *queue) static void indexer_queue_request_status_int(struct indexer_queue *queue, struct indexer_request *request, - int percentage) + const struct indexer_status *status) { void *context; unsigned int i; for (i = 0; i < request->working_context_idx; i++) { context = array_idx_elem(&request->contexts, i); - queue->callback(percentage, context); + queue->callback(status, context); } } void indexer_queue_request_status(struct indexer_queue *queue, struct indexer_request *request, - int percentage) + const struct indexer_status *status) { - i_assert(percentage >= 0 && percentage < 100); - - indexer_queue_request_status_int(queue, request, percentage); + i_assert(status->state == INDEXER_STATE_PROCESSING); + indexer_queue_request_status_int(queue, request, status); } void indexer_queue_move_head_to_tail(struct indexer_queue *queue) @@ -237,14 +236,16 @@ void indexer_queue_request_work(struct indexer_request *request) void indexer_queue_request_finish(struct indexer_queue *queue, struct indexer_request **_request, - bool success) + enum indexer_state state) { struct indexer_request *first_request, *request = *_request; char *first_username; *_request = NULL; - indexer_queue_request_status_int(queue, request, success ? 100 : -1); + i_assert(state != INDEXER_STATE_PROCESSING); + struct indexer_status status = { .state = state }; + indexer_queue_request_status_int(queue, request, &status); if (request->reindex_head || request->reindex_tail) { i_assert(request->working); @@ -292,7 +293,7 @@ indexer_queue_request_cancel(struct indexer_queue *queue, *_request = NULL; request->reindex_head = request->reindex_tail = FALSE; DLLIST2_REMOVE(&queue->head, &queue->tail, request); - indexer_queue_request_finish(queue, &request, FALSE); + indexer_queue_request_finish(queue, &request, INDEXER_STATE_FAILED); } void indexer_queue_cancel(struct indexer_queue *queue, const char *username, diff --git a/src/indexer/indexer-queue.h b/src/indexer/indexer-queue.h index 066173bbd7..5b0a8821c7 100644 --- a/src/indexer/indexer-queue.h +++ b/src/indexer/indexer-queue.h @@ -3,7 +3,8 @@ #include "indexer.h" -typedef void indexer_queue_callback_t(int status, void *context); +typedef void +indexer_queue_callback_t(const struct indexer_status *status, void *context); enum indexer_request_type { /* index messages in the mailbox */ @@ -72,7 +73,7 @@ void indexer_queue_request_remove(struct indexer_queue *queue); /* Give a status update about how far the indexing is going on. */ void indexer_queue_request_status(struct indexer_queue *queue, struct indexer_request *request, - int percentage); + const struct indexer_status *status); /* Move the next request to the end of the queue. */ void indexer_queue_move_head_to_tail(struct indexer_queue *queue); /* Start working on a request */ @@ -80,7 +81,7 @@ void indexer_queue_request_work(struct indexer_request *request); /* Finish the request and free its memory. */ void indexer_queue_request_finish(struct indexer_queue *queue, struct indexer_request **request, - bool success); + enum indexer_state state); /* Iterate through all requests. First it returns the requests currently being worked on, followed by the queued requests in the priority order. If diff --git a/src/indexer/indexer.c b/src/indexer/indexer.c index 12f7065a97..4462c80f61 100644 --- a/src/indexer/indexer.c +++ b/src/indexer/indexer.c @@ -13,8 +13,8 @@ static const struct master_service_settings *set; static struct indexer_queue *queue; -static void -worker_status_callback(int percentage, struct indexer_request *request); +static void worker_status_callback(const struct indexer_status *status, + struct indexer_request *request); static void worker_avail_callback(void); void indexer_refresh_proctitle(void) @@ -84,17 +84,15 @@ static void queue_listen_callback(struct indexer_queue *queue) queue_try_send_more(queue); } -static void -worker_status_callback(int percentage, struct indexer_request *request) +static void worker_status_callback(const struct indexer_status *status, + struct indexer_request *request) { - if (percentage >= 0 && percentage < 100) { - indexer_queue_request_status(queue, request, - percentage); + if (status->state == INDEXER_STATE_PROCESSING) { + indexer_queue_request_status(queue, request, status); return; } - indexer_queue_request_finish(queue, &request, - percentage == 100); + indexer_queue_request_finish(queue, &request, status->state); } static void worker_avail_callback(void) diff --git a/src/indexer/indexer.h b/src/indexer/indexer.h index 17234e50f5..e18f41f38d 100644 --- a/src/indexer/indexer.h +++ b/src/indexer/indexer.h @@ -5,9 +5,9 @@ struct indexer_request; -/* percentage: -1 = failed, 0..99 = indexing in progress, 100 = done */ typedef void -indexer_status_callback_t(int percentage, struct indexer_request *request); +indexer_status_callback_t(const struct indexer_status *status, + struct indexer_request *request); void indexer_refresh_proctitle(void); diff --git a/src/indexer/test-indexer-queue.c b/src/indexer/test-indexer-queue.c index d2fae4df78..b62ba78201 100644 --- a/src/indexer/test-indexer-queue.c +++ b/src/indexer/test-indexer-queue.c @@ -7,7 +7,8 @@ void indexer_refresh_proctitle(void) { } static void -indexer_queue_status_callback(int status ATTR_UNUSED, void *context ATTR_UNUSED) +indexer_queue_status_callback(const struct indexer_status *status ATTR_UNUSED, + void *context ATTR_UNUSED) { } @@ -45,7 +46,7 @@ static void test_indexer_queue(void) test_assert_strcmp_idx(request->mailbox, expected[i].mailbox, i); indexer_queue_request_remove(queue); - indexer_queue_request_finish(queue, &request, TRUE); + indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED); } test_assert(indexer_queue_request_peek(queue) == NULL); @@ -68,7 +69,7 @@ static void test_indexer_queue_repeated_prepend(void) request = indexer_queue_request_peek(queue); indexer_queue_request_remove(queue); - indexer_queue_request_finish(queue, &request, TRUE); + indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED); test_assert(indexer_queue_request_peek(queue) == NULL); @@ -101,7 +102,7 @@ static void test_indexer_queue_reindex(void) test_assert(request->reindex_head); /* finish the request, and it should now be at the head again */ - indexer_queue_request_finish(queue, &request, TRUE); + indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED); request = indexer_queue_request_peek(queue); test_assert_strcmp(request->mailbox, "mailbox1"); test_assert(!request->working); @@ -114,17 +115,17 @@ static void test_indexer_queue_reindex(void) test_assert(request->reindex_tail); /* finish the request, and it should now be at the tail again */ - indexer_queue_request_finish(queue, &request, TRUE); + indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED); request = indexer_queue_request_peek(queue); test_assert_strcmp(request->mailbox, "mailbox2"); indexer_queue_request_remove(queue); - indexer_queue_request_finish(queue, &request, TRUE); + indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED); request = indexer_queue_request_peek(queue); test_assert_strcmp(request->mailbox, "mailbox1"); indexer_queue_request_remove(queue); - indexer_queue_request_finish(queue, &request, TRUE); + indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED); test_assert(indexer_queue_request_peek(queue) == NULL); @@ -179,7 +180,7 @@ static void test_indexer_queue_cancel(void) test_assert(request->reindex_tail); indexer_queue_cancel(queue, "user1", NULL); test_assert(!request->reindex_tail); - indexer_queue_request_finish(queue, &request, TRUE); + indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED); test_assert(indexer_queue_request_peek(queue) == NULL); /* test cancelling mailbox wildcards */ @@ -247,8 +248,8 @@ static void test_indexer_queue_iter(void) indexer_queue_iter_deinit(&iter); /* Finish cleanup */ - indexer_queue_request_finish(queue, &request1, FALSE); - indexer_queue_request_finish(queue, &request2, FALSE); + indexer_queue_request_finish(queue, &request1, INDEXER_STATE_FAILED); + indexer_queue_request_finish(queue, &request2, INDEXER_STATE_FAILED); indexer_queue_cancel_all(queue); test_assert(indexer_queue_request_peek(queue) == NULL); diff --git a/src/indexer/worker-connection.c b/src/indexer/worker-connection.c index 6566edd8eb..06f77fb923 100644 --- a/src/indexer/worker-connection.c +++ b/src/indexer/worker-connection.c @@ -37,11 +37,11 @@ static unsigned int worker_last_process_limit = 0; static struct connection_list *worker_connections; static void worker_connection_call_callback(struct worker_connection *worker, - int percentage) + const struct indexer_status *status) { if (worker->request != NULL) - worker->callback(percentage, worker->request); - if (percentage < 0 || percentage == 100) + worker->callback(status, worker->request); + if (status->state != INDEXER_STATE_PROCESSING) worker->request = NULL; } @@ -50,7 +50,8 @@ static void worker_connection_destroy(struct connection *conn) struct worker_connection *worker = container_of(conn, struct worker_connection, conn); - worker_connection_call_callback(worker, -1); + struct indexer_status status = { .state = INDEXER_STATE_FAILED }; + worker_connection_call_callback(worker, &status); i_free_and_null(worker->request_username); connection_deinit(conn); @@ -120,20 +121,12 @@ worker_connection_input_args(struct connection *conn, const char *const *args) } } - unsigned int percentage; - switch (state) { - case INDEXER_STATE_FAILED: - percentage = -1; - break; - case INDEXER_STATE_COMPLETED: - percentage = 100; - break; - case INDEXER_STATE_PROCESSING: - percentage = total == 0 ? 0 : progress * 100 / total; - break; - } - - worker_connection_call_callback(worker, percentage); + struct indexer_status status = { + .state = state, + .progress = progress, + .total = total, + }; + worker_connection_call_callback(worker, &status); if (worker->request == NULL) { /* disconnect after each request */ ret = -1; diff --git a/src/plugins/fts/fts-indexer.c b/src/plugins/fts/fts-indexer.c index 540bf3746b..e124ae0855 100644 --- a/src/plugins/fts/fts-indexer.c +++ b/src/plugins/fts/fts-indexer.c @@ -14,6 +14,7 @@ #include "fts-api.h" #include "fts-storage.h" #include "fts-indexer.h" +#include "fts-indexer-status.h" #define INDEXER_SOCKET_NAME "indexer" #define INDEXER_WAIT_MSECS 250 @@ -25,7 +26,7 @@ struct fts_indexer_context { struct ioloop *ioloop; struct timeval search_start_time, last_notify; - unsigned int percentage; + struct indexer_status status; struct connection_list *connection_list; bool notified:1; @@ -55,8 +56,8 @@ static void fts_indexer_notify(struct fts_indexer_context *ctx) struct mail_storage_progress_details dtl = { .verb = "Indexed", - .total = 100, - .processed = ctx->percentage, + .total = ctx->status.total, + .processed = ctx->status.progress, .start_time = ctx->search_start_time, .now = ioloop_timeval, }; @@ -145,7 +146,6 @@ fts_indexer_input_args(struct connection *conn, const char *const *args) { struct fts_indexer_context *ctx = container_of(conn, struct fts_indexer_context, conn); - int percentage; if (args[1] == NULL) { e_error(conn->event, "indexer sent invalid reply"); return -1; @@ -156,19 +156,38 @@ fts_indexer_input_args(struct connection *conn, const char *const *args) } if (strcmp(args[1], "OK") == 0) return 1; - if (str_to_int(args[1], &percentage) < 0) { - e_error(conn->event, "indexer sent invalid progress: %s", args[1]); + if (str_to_int(args[1], &ctx->status.state) < 0) { + e_error(conn->event, "indexer sent invalid percentage: %s", args[1]); ctx->failed = TRUE; return -1; } - if (percentage < 0) { + if (ctx->status.state < INDEXER_STATE_FAILED || + ctx->status.state > INDEXER_STATE_COMPLETED) { + e_error(conn->event, "indexer sent invalid state: %s", args[2]); + ctx->failed = TRUE; + return -1; + } + if (ctx->status.state == INDEXER_STATE_FAILED) { e_error(ctx->box->event, "indexer failed to index mailbox"); ctx->failed = TRUE; return -1; } - ctx->percentage = percentage; + if (str_array_length(args) < 4) { + e_error(conn->event, "indexer sent invalid reply"); + return -1; + } + if (str_to_uint32(args[2], &ctx->status.progress) < 0) { + e_error(conn->event, "indexer sent invalid processed: %s", args[2]); + ctx->failed = TRUE; + return -1; + } + if (str_to_uint32(args[3], &ctx->status.total) < 0) { + e_error(conn->event, "indexer sent invalid total: %s", args[3]); + ctx->failed = TRUE; + return -1; + } time_t elapsed = ioloop_time - ctx->search_start_time.tv_sec; - if (ctx->percentage == 100) + if (ctx->status.state == INDEXER_STATE_COMPLETED) ctx->completed = TRUE; else if (ctx->conn.input_idle_timeout_secs > 0 && elapsed > ctx->conn.input_idle_timeout_secs) { -- 2.47.3