indexer: Provide progress updates in messages counts rather than just percentage...

diff --git a/src/indexer/indexer-client.c b/src/indexer/indexer-client.c
index 8bc9080beb9321a11300a779156779a432fc5567..c9ed95c9da0740c46db2c45fa9f58b9bb066a9d0 100644 (file)
--- a/src/indexer/indexer-client.c
+++ b/src/indexer/indexer-client.c
@@ -253,15 +253,20 @@ indexer_client_input_args(struct connection *conn, const char *const *args)
        return 1;
 }
 
-void indexer_client_status_callback(int percentage, void *context)
+void indexer_client_status_callback(const struct indexer_status *status,
+                                   void *context)
 {
        struct indexer_client_request *ctx = context;
 
        if (ctx->client->conn.output != NULL) T_BEGIN {
-               o_stream_nsend_str(ctx->client->conn.output,
-                       t_strdup_printf("%u\t%d\n", ctx->tag, percentage));
+               const char *update = t_strdup_printf(
+                       "%u\t%d\t%u\t%u\n", ctx->tag,
+                       status->state, status->progress, status->total);
+               o_stream_nsend_str(ctx->client->conn.output, update);
+
        } T_END;
-       if (percentage < 0 || percentage == 100) {
+
+       if (status->state != INDEXER_STATE_PROCESSING) {
                indexer_client_unref(ctx->client);
                i_free(ctx);
        }

diff --git a/src/indexer/indexer-client.h b/src/indexer/indexer-client.h
index 609c67805abdecf52c5106e94aaf4e593d55fe5e..99efa413f3bf8145d879a0b15adc412f65c36595 100644 (file)
--- a/src/indexer/indexer-client.h
+++ b/src/indexer/indexer-client.h
@@ -1,11 +1,13 @@
 #ifndef INDEXER_CLIENT_H
 #define INDEXER_CLIENT_H
 
+struct indexer_status;
 struct indexer_queue;
 
 void indexer_client_create(struct master_service_connection *conn,
                           struct indexer_queue *queue);
-void indexer_client_status_callback(int percentage, void *context);
+void indexer_client_status_callback(const struct indexer_status *status,
+                                   void *context);
 
 unsigned int indexer_clients_get_count(void);
 void indexer_clients_destroy_all(void);

diff --git a/src/indexer/indexer-queue.c b/src/indexer/indexer-queue.c
index 5df538bc783f41fc4e5d17ca76ebe4fb4697e391..ee29fa3351d17f0d875c978920eeb2e8d3acba6f 100644 (file)
--- a/src/indexer/indexer-queue.c
+++ b/src/indexer/indexer-queue.c
@@ -199,24 +199,23 @@ void indexer_queue_request_remove(struct indexer_queue *queue)
 
 static void indexer_queue_request_status_int(struct indexer_queue *queue,
                                             struct indexer_request *request,
-                                            int percentage)
+                                            const struct indexer_status *status)
 {
        void *context;
        unsigned int i;
 
        for (i = 0; i < request->working_context_idx; i++) {
                context = array_idx_elem(&request->contexts, i);
-               queue->callback(percentage, context);
+               queue->callback(status, context);
        }
 }
 
 void indexer_queue_request_status(struct indexer_queue *queue,
                                  struct indexer_request *request,
-                                 int percentage)
+                                 const struct indexer_status *status)
 {
-       i_assert(percentage >= 0 && percentage < 100);
-
-       indexer_queue_request_status_int(queue, request, percentage);
+       i_assert(status->state == INDEXER_STATE_PROCESSING);
+       indexer_queue_request_status_int(queue, request, status);
 }
 
 void indexer_queue_move_head_to_tail(struct indexer_queue *queue)
@@ -237,14 +236,16 @@ void indexer_queue_request_work(struct indexer_request *request)
 
 void indexer_queue_request_finish(struct indexer_queue *queue,
                                  struct indexer_request **_request,
-                                 bool success)
+                                 enum indexer_state state)
 {
        struct indexer_request *first_request, *request = *_request;
        char *first_username;
 
        *_request = NULL;
 
-       indexer_queue_request_status_int(queue, request, success ? 100 : -1);
+       i_assert(state != INDEXER_STATE_PROCESSING);
+       struct indexer_status status = { .state = state };
+       indexer_queue_request_status_int(queue, request, &status);
 
        if (request->reindex_head || request->reindex_tail) {
                i_assert(request->working);
@@ -292,7 +293,7 @@ indexer_queue_request_cancel(struct indexer_queue *queue,
        *_request = NULL;
        request->reindex_head = request->reindex_tail = FALSE;
        DLLIST2_REMOVE(&queue->head, &queue->tail, request);
-       indexer_queue_request_finish(queue, &request, FALSE);
+       indexer_queue_request_finish(queue, &request, INDEXER_STATE_FAILED);
 }
 
 void indexer_queue_cancel(struct indexer_queue *queue, const char *username,

diff --git a/src/indexer/indexer-queue.h b/src/indexer/indexer-queue.h
index 066173bbd7451afb9fb6835774f18a2fa007453c..5b0a8821c7d6216f882610a7a843f32b97b7a8fa 100644 (file)
--- a/src/indexer/indexer-queue.h
+++ b/src/indexer/indexer-queue.h
@@ -3,7 +3,8 @@
 
 #include "indexer.h"
 
-typedef void indexer_queue_callback_t(int status, void *context);
+typedef void
+indexer_queue_callback_t(const struct indexer_status *status, void *context);
 
 enum indexer_request_type {
        /* index messages in the mailbox */
@@ -72,7 +73,7 @@ void indexer_queue_request_remove(struct indexer_queue *queue);
 /* Give a status update about how far the indexing is going on. */
 void indexer_queue_request_status(struct indexer_queue *queue,
                                  struct indexer_request *request,
-                                 int percentage);
+                                 const struct indexer_status *status);
 /* Move the next request to the end of the queue. */
 void indexer_queue_move_head_to_tail(struct indexer_queue *queue);
 /* Start working on a request */
@@ -80,7 +81,7 @@ void indexer_queue_request_work(struct indexer_request *request);
 /* Finish the request and free its memory. */
 void indexer_queue_request_finish(struct indexer_queue *queue,
                                  struct indexer_request **request,
-                                 bool success);
+                                 enum indexer_state state);
 
 /* Iterate through all requests. First it returns the requests currently being
    worked on, followed by the queued requests in the priority order. If

diff --git a/src/indexer/indexer.c b/src/indexer/indexer.c
index 12f7065a973751d156ff1f44e20651c5f28d2f78..4462c80f6101ca08934e6e88b0211e79824aa0d3 100644 (file)
--- a/src/indexer/indexer.c
+++ b/src/indexer/indexer.c
@@ -13,8 +13,8 @@
 static const struct master_service_settings *set;
 static struct indexer_queue *queue;
 
-static void
-worker_status_callback(int percentage, struct indexer_request *request);
+static void worker_status_callback(const struct indexer_status *status,
+                                  struct indexer_request *request);
 static void worker_avail_callback(void);
 
 void indexer_refresh_proctitle(void)
@@ -84,17 +84,15 @@ static void queue_listen_callback(struct indexer_queue *queue)
        queue_try_send_more(queue);
 }
 
-static void
-worker_status_callback(int percentage, struct indexer_request *request)
+static void worker_status_callback(const struct indexer_status *status,
+                                  struct indexer_request *request)
 {
-       if (percentage >= 0 && percentage < 100) {
-               indexer_queue_request_status(queue, request,
-                                            percentage);
+       if (status->state == INDEXER_STATE_PROCESSING) {
+               indexer_queue_request_status(queue, request, status);
                return;
        }
 
-       indexer_queue_request_finish(queue, &request,
-                                    percentage == 100);
+       indexer_queue_request_finish(queue, &request, status->state);
 }
 
 static void worker_avail_callback(void)

diff --git a/src/indexer/indexer.h b/src/indexer/indexer.h
index 17234e50f5864cdee5d77f9bb075881544f85bb9..e18f41f38dee39f9c6619cf6506095fc7c2ac720 100644 (file)
--- a/src/indexer/indexer.h
+++ b/src/indexer/indexer.h
@@ -5,9 +5,9 @@
 
 struct indexer_request;
 
-/* percentage: -1 = failed, 0..99 = indexing in progress, 100 = done */
 typedef void
-indexer_status_callback_t(int percentage, struct indexer_request *request);
+indexer_status_callback_t(const struct indexer_status *status,
+                          struct indexer_request *request);
 
 void indexer_refresh_proctitle(void);
 

diff --git a/src/indexer/test-indexer-queue.c b/src/indexer/test-indexer-queue.c
index d2fae4df783af78990d0502c14fbfaab00d974d2..b62ba78201a9ade3eae3635be0580387c38a0679 100644 (file)
--- a/src/indexer/test-indexer-queue.c
+++ b/src/indexer/test-indexer-queue.c
@@ -7,7 +7,8 @@
 void indexer_refresh_proctitle(void) { }
 
 static void
-indexer_queue_status_callback(int status ATTR_UNUSED, void *context ATTR_UNUSED)
+indexer_queue_status_callback(const struct indexer_status *status ATTR_UNUSED,
+                             void *context ATTR_UNUSED)
 {
 }
 
@@ -45,7 +46,7 @@ static void test_indexer_queue(void)
                test_assert_strcmp_idx(request->mailbox, expected[i].mailbox, i);
 
                indexer_queue_request_remove(queue);
-               indexer_queue_request_finish(queue, &request, TRUE);
+               indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED);
        }
        test_assert(indexer_queue_request_peek(queue) == NULL);
 
@@ -68,7 +69,7 @@ static void test_indexer_queue_repeated_prepend(void)
 
        request = indexer_queue_request_peek(queue);
        indexer_queue_request_remove(queue);
-       indexer_queue_request_finish(queue, &request, TRUE);
+       indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED);
 
        test_assert(indexer_queue_request_peek(queue) == NULL);
 
@@ -101,7 +102,7 @@ static void test_indexer_queue_reindex(void)
        test_assert(request->reindex_head);
 
        /* finish the request, and it should now be at the head again */
-       indexer_queue_request_finish(queue, &request, TRUE);
+       indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED);
        request = indexer_queue_request_peek(queue);
        test_assert_strcmp(request->mailbox, "mailbox1");
        test_assert(!request->working);
@@ -114,17 +115,17 @@ static void test_indexer_queue_reindex(void)
        test_assert(request->reindex_tail);
 
        /* finish the request, and it should now be at the tail again */
-       indexer_queue_request_finish(queue, &request, TRUE);
+       indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED);
 
        request = indexer_queue_request_peek(queue);
        test_assert_strcmp(request->mailbox, "mailbox2");
        indexer_queue_request_remove(queue);
-       indexer_queue_request_finish(queue, &request, TRUE);
+       indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED);
 
        request = indexer_queue_request_peek(queue);
        test_assert_strcmp(request->mailbox, "mailbox1");
        indexer_queue_request_remove(queue);
-       indexer_queue_request_finish(queue, &request, TRUE);
+       indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED);
 
        test_assert(indexer_queue_request_peek(queue) == NULL);
 
@@ -179,7 +180,7 @@ static void test_indexer_queue_cancel(void)
        test_assert(request->reindex_tail);
        indexer_queue_cancel(queue, "user1", NULL);
        test_assert(!request->reindex_tail);
-       indexer_queue_request_finish(queue, &request, TRUE);
+       indexer_queue_request_finish(queue, &request, INDEXER_STATE_COMPLETED);
        test_assert(indexer_queue_request_peek(queue) == NULL);
 
        /* test cancelling mailbox wildcards */
@@ -247,8 +248,8 @@ static void test_indexer_queue_iter(void)
        indexer_queue_iter_deinit(&iter);
 
        /* Finish cleanup */
-       indexer_queue_request_finish(queue, &request1, FALSE);
-       indexer_queue_request_finish(queue, &request2, FALSE);
+       indexer_queue_request_finish(queue, &request1, INDEXER_STATE_FAILED);
+       indexer_queue_request_finish(queue, &request2, INDEXER_STATE_FAILED);
 
        indexer_queue_cancel_all(queue);
        test_assert(indexer_queue_request_peek(queue) == NULL);

diff --git a/src/indexer/worker-connection.c b/src/indexer/worker-connection.c
index 6566edd8eb795b58d80f5f686396e0364a7f4646..06f77fb92313abe8de330495db22aba7c2680d54 100644 (file)
--- a/src/indexer/worker-connection.c
+++ b/src/indexer/worker-connection.c
@@ -37,11 +37,11 @@ static unsigned int worker_last_process_limit = 0;
 static struct connection_list *worker_connections;
 
 static void worker_connection_call_callback(struct worker_connection *worker,
-                                           int percentage)
+                                           const struct indexer_status *status)
 {
        if (worker->request != NULL)
-               worker->callback(percentage, worker->request);
-       if (percentage < 0 || percentage == 100)
+               worker->callback(status, worker->request);
+       if (status->state != INDEXER_STATE_PROCESSING)
                worker->request = NULL;
 }
 
@@ -50,7 +50,8 @@ static void worker_connection_destroy(struct connection *conn)
        struct worker_connection *worker =
                container_of(conn, struct worker_connection, conn);
 
-       worker_connection_call_callback(worker, -1);
+       struct indexer_status status = { .state = INDEXER_STATE_FAILED };
+       worker_connection_call_callback(worker, &status);
        i_free_and_null(worker->request_username);
        connection_deinit(conn);
 
@@ -120,20 +121,12 @@ worker_connection_input_args(struct connection *conn, const char *const *args)
                }
        }
 
-       unsigned int percentage;
-       switch (state) {
-       case INDEXER_STATE_FAILED:
-               percentage = -1;
-               break;
-       case INDEXER_STATE_COMPLETED:
-               percentage = 100;
-               break;
-       case INDEXER_STATE_PROCESSING:
-               percentage = total == 0 ? 0 : progress * 100 / total;
-               break;
-       }
-
-       worker_connection_call_callback(worker, percentage);
+       struct indexer_status status = {
+               .state = state,
+               .progress = progress,
+               .total = total,
+       };
+       worker_connection_call_callback(worker, &status);
        if (worker->request == NULL) {
                /* disconnect after each request */
                ret = -1;

diff --git a/src/plugins/fts/fts-indexer.c b/src/plugins/fts/fts-indexer.c
index 540bf3746bf77989c1aaab6a5e3f3fbf669b9095..e124ae08554b1ffd511797177cbf36b2e2bce36d 100644 (file)
--- a/src/plugins/fts/fts-indexer.c
+++ b/src/plugins/fts/fts-indexer.c
@@ -14,6 +14,7 @@
 #include "fts-api.h"
 #include "fts-storage.h"
 #include "fts-indexer.h"
+#include "fts-indexer-status.h"
 
 #define INDEXER_SOCKET_NAME "indexer"
 #define INDEXER_WAIT_MSECS 250
@@ -25,7 +26,7 @@ struct fts_indexer_context {
        struct ioloop *ioloop;
 
        struct timeval search_start_time, last_notify;
-       unsigned int percentage;
+       struct indexer_status status;
        struct connection_list *connection_list;
 
        bool notified:1;
@@ -55,8 +56,8 @@ static void fts_indexer_notify(struct fts_indexer_context *ctx)
 
        struct mail_storage_progress_details dtl = {
                .verb = "Indexed",
-               .total = 100,
-               .processed = ctx->percentage,
+               .total = ctx->status.total,
+               .processed = ctx->status.progress,
                .start_time = ctx->search_start_time,
                .now = ioloop_timeval,
        };
@@ -145,7 +146,6 @@ fts_indexer_input_args(struct connection *conn, const char *const *args)
 {
        struct fts_indexer_context *ctx =
                container_of(conn, struct fts_indexer_context, conn);
-       int percentage;
        if (args[1] == NULL) {
                e_error(conn->event, "indexer sent invalid reply");
                return -1;
@@ -156,19 +156,38 @@ fts_indexer_input_args(struct connection *conn, const char *const *args)
        }
        if (strcmp(args[1], "OK") == 0)
                return 1;
-       if (str_to_int(args[1], &percentage) < 0) {
-               e_error(conn->event, "indexer sent invalid progress: %s", args[1]);
+       if (str_to_int(args[1], &ctx->status.state) < 0) {
+               e_error(conn->event, "indexer sent invalid percentage: %s", args[1]);
                ctx->failed = TRUE;
                return -1;
        }
-       if (percentage < 0) {
+       if (ctx->status.state < INDEXER_STATE_FAILED ||
+           ctx->status.state > INDEXER_STATE_COMPLETED) {
+               e_error(conn->event, "indexer sent invalid state: %s", args[2]);
+               ctx->failed = TRUE;
+               return -1;
+       }
+       if (ctx->status.state == INDEXER_STATE_FAILED) {
                e_error(ctx->box->event, "indexer failed to index mailbox");
                ctx->failed = TRUE;
                return -1;
        }
-       ctx->percentage = percentage;
+       if (str_array_length(args) < 4) {
+               e_error(conn->event, "indexer sent invalid reply");
+               return -1;
+       }
+       if (str_to_uint32(args[2], &ctx->status.progress) < 0) {
+               e_error(conn->event, "indexer sent invalid processed: %s", args[2]);
+               ctx->failed = TRUE;
+               return -1;
+       }
+       if (str_to_uint32(args[3], &ctx->status.total) < 0) {
+               e_error(conn->event, "indexer sent invalid total: %s", args[3]);
+               ctx->failed = TRUE;
+               return -1;
+       }
        time_t elapsed = ioloop_time - ctx->search_start_time.tv_sec;
-       if (ctx->percentage == 100)
+       if (ctx->status.state == INDEXER_STATE_COMPLETED)
                ctx->completed = TRUE;
        else if (ctx->conn.input_idle_timeout_secs > 0 &&
                 elapsed > ctx->conn.input_idle_timeout_secs) {