taskprocessor: Add high water mark warnings

If a taskprocessor's queue grows large, this can indicate that there
may be a problem with tasks not leaving the processor or else that
the number of available task processors for a given type of task is
too low. This patch makes it so that if a taskprocessor's task queue
grows above 100 queued tasks that it will emit a warning message.
Warning messages are emitted only once per task processor.

ASTERISK-25518 #close
Reported by: Jonathan Rose

Change-Id: Ib1607c35d18c1d6a0575b3f0e3ff5d932fd6600c

diff --git a/main/taskprocessor.c b/main/taskprocessor.c
index e8dc8f5f23e768c0eb3eaa4f2b506225d3effd4b..f382814af9796f4beb13321f98ea2814c9a793ab 100644 (file)
--- a/main/taskprocessor.c
+++ b/main/taskprocessor.c
@@ -83,6 +83,8 @@ struct ast_taskprocessor {
        pthread_t thread;
        /*! Indicates if the taskprocessor is currently executing a task */
        unsigned int executing:1;
+       /*! Indicates that a high water warning has been issued on this task processor */
+       unsigned int high_water_warned:1;
 };
 
 /*!
@@ -714,6 +716,8 @@ void *ast_taskprocessor_unreference(struct ast_taskprocessor *tps)
        return NULL;
 }
 
+#define HIGH_WATER_LEVEL 100
+
 /* push the task into the taskprocessor queue */
 static int taskprocessor_push(struct ast_taskprocessor *tps, struct tps_task *t)
 {
@@ -733,6 +737,13 @@ static int taskprocessor_push(struct ast_taskprocessor *tps, struct tps_task *t)
        ao2_lock(tps);
        AST_LIST_INSERT_TAIL(&tps->tps_queue, t, list);
        previous_size = tps->tps_queue_size++;
+
+       if (previous_size >= HIGH_WATER_LEVEL && !tps->high_water_warned) {
+               ast_log(LOG_WARNING, "The '%s' task processor queue reached %d scheduled tasks.\n",
+                       tps->name, previous_size);
+               tps->high_water_warned = 1;
+       }
+
        /* The currently executing task counts as still in queue */
        was_empty = tps->executing ? 0 : previous_size == 0;
        ao2_unlock(tps);