trunk: Split FULL and INACTIVE states

diff --git a/src/lib/server/trunk.c b/src/lib/server/trunk.c
index 9033622c243c8f271704a6608971cd431203b0b2..bd659cd31e9ac42a8a6f621799b4d0dd51c8ce53 100644 (file)
--- a/src/lib/server/trunk.c
+++ b/src/lib/server/trunk.c
@@ -181,9 +181,12 @@ struct fr_trunk_s {
 
        fr_heap_t               *active;                //!< Connections which can service requests.
 
-       fr_dlist_head_t         inactive;               //!< Connections which have too many outstanding
+       fr_dlist_head_t         full;                   //!< Connections which have too many outstanding
                                                        ///< requests.
 
+       fr_dlist_head_t         inactive;               //!< Connections which have been signalled to be
+                                                       ///< inactive by the API client.
+
        fr_dlist_head_t         failed;                 //!< Connections that'll be reconnected shortly.
 
        fr_dlist_head_t         closed;                 //!< Connections that have closed. Either due to
@@ -306,6 +309,7 @@ static fr_table_num_ordered_t const fr_trunk_connection_states[] = {
        { "HALTED",             FR_TRUNK_CONN_HALTED            },
        { "CONNECTING",         FR_TRUNK_CONN_CONNECTING        },
        { "ACTIVE",             FR_TRUNK_CONN_ACTIVE            },
+       { "FULL",               FR_TRUNK_CONN_FULL              },
        { "INACTIVE",           FR_TRUNK_CONN_INACTIVE          },
        { "FAILED",             FR_TRUNK_CONN_FAILED            },
        { "CLOSED",             FR_TRUNK_CONN_CLOSED            },
@@ -583,11 +587,12 @@ static uint32_t trunk_requests_per_connnection(uint16_t *conn_count_out, uint32_
                                               fr_trunk_t *trunk, fr_time_t now);
 
 static int trunk_connection_spawn(fr_trunk_t *trunk, fr_time_t now);
-static inline void trunk_connection_auto_inactive(fr_trunk_connection_t *tconn);
-static inline void trunk_connection_auto_reactivate(fr_trunk_connection_t *tconn);
+static inline void trunk_connection_auto_full(fr_trunk_connection_t *tconn);
+static inline void trunk_connection_auto_unfull(fr_trunk_connection_t *tconn);
 static inline void trunk_connection_readable(fr_trunk_connection_t *tconn);
 static inline void trunk_connection_writable(fr_trunk_connection_t *tconn);
 static void trunk_connection_event_update(fr_trunk_connection_t *tconn);
+static void trunk_connection_enter_full(fr_trunk_connection_t *tconn);
 static void trunk_connection_enter_inactive(fr_trunk_connection_t *tconn);
 static void trunk_connection_enter_draining(fr_trunk_connection_t *tconn);
 static void trunk_connection_enter_draining_to_free(fr_trunk_connection_t *tconn);
@@ -678,9 +683,9 @@ static void trunk_request_remove_from_conn(fr_trunk_request_t *treq)
        DEBUG4("[%" PRIu64 "] Trunk connection released request %" PRIu64, tconn->pub.conn->id, treq->id);
 
        switch (tconn->state){
-       case FR_TRUNK_CONN_INACTIVE:
-               trunk_connection_auto_reactivate(tconn);                /* Check if we can switch back to active */
-               if (tconn->state == FR_TRUNK_CONN_INACTIVE) break;      /* Only fallthrough if conn is now active */
+       case FR_TRUNK_CONN_FULL:
+               trunk_connection_auto_unfull(tconn);            /* Check if we can switch back to active */
+               if (tconn->state == FR_TRUNK_CONN_FULL) break;  /* Only fallthrough if conn is now active */
                /* FALL-THROUGH */
 
        case FR_TRUNK_CONN_ACTIVE:
@@ -823,7 +828,7 @@ static void trunk_request_enter_pending(fr_trunk_request_t *treq, fr_trunk_conne
         *      Check if we need to automatically transition the
         *      connection to full.
         */
-       trunk_connection_auto_inactive(tconn);
+       trunk_connection_auto_full(tconn);
 
        /*
         *      Reorder the connection in the heap now it has an
@@ -1490,6 +1495,7 @@ uint64_t fr_trunk_connection_requests_requeue(fr_trunk_connection_t *tconn, int
 {
        switch (tconn->state) {
        case FR_TRUNK_CONN_ACTIVE:
+       case FR_TRUNK_CONN_FULL:
        case FR_TRUNK_CONN_INACTIVE:
                return trunk_connection_requests_requeue(tconn, states, max, fail_bound);
 
@@ -2081,6 +2087,7 @@ uint16_t fr_trunk_connection_count_by_state(fr_trunk_t *trunk, int conn_state)
 
        if (conn_state & FR_TRUNK_CONN_CONNECTING) count += fr_dlist_num_elements(&trunk->connecting);
        if (conn_state & FR_TRUNK_CONN_ACTIVE) count += fr_heap_num_elements(trunk->active);
+       if (conn_state & FR_TRUNK_CONN_FULL) count += fr_dlist_num_elements(&trunk->full);
        if (conn_state & FR_TRUNK_CONN_INACTIVE) count += fr_dlist_num_elements(&trunk->inactive);
        if (conn_state & FR_TRUNK_CONN_FAILED) count += fr_dlist_num_elements(&trunk->failed);
        if (conn_state & FR_TRUNK_CONN_CLOSED) count += fr_dlist_num_elements(&trunk->closed);
@@ -2115,7 +2122,7 @@ uint32_t fr_trunk_request_count_by_connection(fr_trunk_connection_t const *tconn
  *
  * @param[in] tconn    to potentially mark as inactive.
  */
-static inline void trunk_connection_auto_inactive(fr_trunk_connection_t *tconn)
+static inline void trunk_connection_auto_full(fr_trunk_connection_t *tconn)
 {
        fr_trunk_t      *trunk = tconn->pub.trunk;
        uint32_t        count;
@@ -2127,32 +2134,43 @@ static inline void trunk_connection_auto_inactive(fr_trunk_connection_t *tconn)
         */
        if (trunk->conf.max_req_per_conn > 0) {
                count = fr_trunk_request_count_by_connection(tconn, FR_TRUNK_REQUEST_STATE_ALL);
-               if (count >= trunk->conf.max_req_per_conn) trunk_connection_enter_inactive(tconn);
+               if (count >= trunk->conf.max_req_per_conn) trunk_connection_enter_full(tconn);
        }
 }
 
-/** Automatically mark a connection as active or reconnect it
+/** Return whether a trunk connection should currently be considered full
  *
- * @param[in] tconn    to potentially mark as active or reconnect.
+ * @param[in] tconn    to check.
+ * @return
+ *     - true if the connection is full.
+ *     - false if the connection is not full.
  */
-static inline void trunk_connection_auto_reactivate(fr_trunk_connection_t *tconn)
+static inline bool trunk_connection_is_full(fr_trunk_connection_t *tconn)
 {
        fr_trunk_t      *trunk = tconn->pub.trunk;
        uint32_t        count;
 
        /*
-        *      Externally signalled that the connection should
-        *      be kept inactive.
+        *      Enforces max_req_per_conn
         */
-       if (tconn->signalled_inactive) return;
+       count = fr_trunk_request_count_by_connection(tconn, FR_TRUNK_REQUEST_STATE_ALL);
+       if ((trunk->conf.max_req_per_conn == 0) || (count < trunk->conf.max_req_per_conn)) return false;
+
+       return true;
+}
+
+/** Automatically mark a connection as active or reconnect it
+ *
+ * @param[in] tconn    to potentially mark as active or reconnect.
+ */
+static inline void trunk_connection_auto_unfull(fr_trunk_connection_t *tconn)
+{
+       if (tconn->state != FR_TRUNK_CONN_FULL) return;
 
        /*
         *      Enforces max_req_per_conn
         */
-       count = fr_trunk_request_count_by_connection(tconn, FR_TRUNK_REQUEST_STATE_ALL);
-       if ((trunk->conf.max_req_per_conn == 0) || (count < trunk->conf.max_req_per_conn)) {
-               trunk_connection_enter_active(tconn);
-       }
+       if (!trunk_connection_is_full(tconn)) trunk_connection_enter_active(tconn);
 }
 
 /** A connection is readable.  Call the request_demux function to read pending requests
@@ -2205,6 +2223,7 @@ static void trunk_connection_event_update(fr_trunk_connection_t *tconn)
         *      requests.
         */
        case FR_TRUNK_CONN_ACTIVE:
+       case FR_TRUNK_CONN_FULL:
        case FR_TRUNK_CONN_INACTIVE:
        case FR_TRUNK_CONN_DRAINING:
        case FR_TRUNK_CONN_DRAINING_TO_FREE:
@@ -2248,6 +2267,30 @@ static void trunk_connection_event_update(fr_trunk_connection_t *tconn)
  * Called whenever a trunk connection is at the maximum number of requests.
  * Removes the connection from the connected heap, and places it in the full list.
  */
+static void trunk_connection_enter_full(fr_trunk_connection_t *tconn)
+{
+       fr_trunk_t              *trunk = tconn->pub.trunk;
+       int                     ret;
+
+       switch (tconn->state) {
+       case FR_TRUNK_CONN_ACTIVE:
+               ret = fr_heap_extract(trunk->active, tconn);
+               if (!fr_cond_assert(ret == 0)) return;
+               break;
+
+       default:
+               if (!fr_cond_assert(0)) return;
+       }
+
+       fr_dlist_insert_head(&trunk->full, tconn);
+       CONN_STATE_TRANSITION(FR_TRUNK_CONN_FULL);
+}
+
+/** Transition a connection to the inactive state
+ *
+ * Called whenever the API client wants to stop new requests being enqueued
+ * on a trunk connection.
+ */
 static void trunk_connection_enter_inactive(fr_trunk_connection_t *tconn)
 {
        fr_trunk_t              *trunk = tconn->pub.trunk;
@@ -2259,6 +2302,10 @@ static void trunk_connection_enter_inactive(fr_trunk_connection_t *tconn)
                if (!fr_cond_assert(ret == 0)) return;
                break;
 
+       case FR_TRUNK_CONN_FULL:
+               fr_dlist_remove(&trunk->full, tconn);
+               break;
+
        default:
                if (!fr_cond_assert(0)) return;
        }
@@ -2283,6 +2330,10 @@ static void trunk_connection_enter_draining(fr_trunk_connection_t *tconn)
                if (!fr_cond_assert(ret == 0)) return;
                break;
 
+       case FR_TRUNK_CONN_FULL:
+               fr_dlist_remove(&trunk->full, tconn);
+               break;
+
        case FR_TRUNK_CONN_INACTIVE:
                fr_dlist_remove(&trunk->inactive, tconn);
                break;
@@ -2318,6 +2369,10 @@ static void trunk_connection_enter_draining_to_free(fr_trunk_connection_t *tconn
                if (!fr_cond_assert(ret == 0)) return;
                break;
 
+       case FR_TRUNK_CONN_FULL:
+               fr_dlist_remove(&trunk->full, tconn);
+               break;
+
        case FR_TRUNK_CONN_INACTIVE:
                fr_dlist_remove(&trunk->inactive, tconn);
                break;
@@ -2344,15 +2399,18 @@ static void trunk_connection_enter_draining_to_free(fr_trunk_connection_t *tconn
 
 /** Transition a connection back to the active state
  *
- * This should only be called on a connection which is in the full state.
- * This is *NOT* to signal the a connection has just become active from the
- * connecting state.
+ * This should only be called on a connection which is in the full state,
+ * inactive state, draining state or connecting state.
  */
 static void trunk_connection_enter_active(fr_trunk_connection_t *tconn)
 {
        fr_trunk_t              *trunk = tconn->pub.trunk;
 
        switch (tconn->state) {
+       case FR_TRUNK_CONN_FULL:
+               fr_dlist_remove(&trunk->full, tconn);
+               break;
+
        case FR_TRUNK_CONN_INACTIVE:
                fr_dlist_remove(&trunk->inactive, tconn);
                break;
@@ -2475,6 +2533,7 @@ static void _trunk_connection_on_shutdown(UNUSED fr_connection_t *conn, UNUSED f
                return;
 
        case FR_TRUNK_CONN_ACTIVE:              /* Transition to draining-to-free */
+       case FR_TRUNK_CONN_FULL:
        case FR_TRUNK_CONN_INACTIVE:
        case FR_TRUNK_CONN_DRAINING:
                break;
@@ -2576,6 +2635,11 @@ static void _trunk_connection_on_closed(UNUSED fr_connection_t *conn, UNUSED fr_
                rad_assert(fr_trunk_request_count_by_connection(tconn, FR_TRUNK_REQUEST_STATE_ALL) == 0);
                break;
 
+       case FR_TRUNK_CONN_FULL:
+               fr_dlist_remove(&trunk->full, tconn);
+               need_requeue = true;
+               break;
+
        case FR_TRUNK_CONN_INACTIVE:
                fr_dlist_remove(&trunk->inactive, tconn);
                need_requeue = true;
@@ -2620,7 +2684,6 @@ static void _trunk_connection_on_closed(UNUSED fr_connection_t *conn, UNUSED fr_
         *      Clear statistics and flags
         */
        tconn->sent_count = 0;
-       tconn->signalled_inactive = false;
 
        /*
         *      Remove the I/O events
@@ -2689,6 +2752,10 @@ static void _trunk_connection_on_halted(UNUSED fr_connection_t *conn, UNUSED fr_
                need_requeue = true;
                break;
 
+       case FR_TRUNK_CONN_FULL:
+               fr_dlist_remove(&trunk->full, tconn);
+               break;
+
        case FR_TRUNK_CONN_INACTIVE:
                fr_dlist_remove(&trunk->inactive, tconn);
                need_requeue = true;
@@ -2993,7 +3060,7 @@ void fr_trunk_connection_signal_inactive(fr_trunk_connection_t *tconn)
 
        switch (tconn->state) {
        case FR_TRUNK_CONN_ACTIVE:
-               tconn->signalled_inactive = true;               /* Prevent tconn from automatically being marked as active */
+       case FR_TRUNK_CONN_FULL:
                trunk_connection_enter_inactive(tconn);
                break;
 
@@ -3008,12 +3075,22 @@ void fr_trunk_connection_signal_inactive(fr_trunk_connection_t *tconn)
  */
 void fr_trunk_connection_signal_active(fr_trunk_connection_t *tconn)
 {
-       /* Can be called anywhere */
-
-       tconn->signalled_inactive = false;                      /* Allow inactive/active state to be changed automatically again */
        switch (tconn->state) {
+       case FR_TRUNK_CONN_FULL:
+               trunk_connection_auto_unfull(tconn);    /* Mark as active if it should be active */
+               break;
+
        case FR_TRUNK_CONN_INACTIVE:
-               trunk_connection_auto_reactivate(tconn);        /* Mark as active if it should be active */
+               /*
+                *      Do the appropriate state transition based on
+                *      how many requests the trunk connection is
+                *      currently servicing.
+                */
+               if (trunk_connection_is_full(tconn)) {
+                       trunk_connection_enter_full(tconn);
+                       break;
+               }
+               trunk_connection_enter_active(tconn);
                break;
 
        default:
@@ -3446,6 +3523,7 @@ do { \
                }
        }
 
+       COUNT_BY_STATE(FR_TRUNK_CONN_FULL, full);
        COUNT_BY_STATE(FR_TRUNK_CONN_INACTIVE, inactive);
        COUNT_BY_STATE(FR_TRUNK_CONN_FAILED, failed);
        COUNT_BY_STATE(FR_TRUNK_CONN_CLOSED, closed);
@@ -3639,6 +3717,7 @@ do { \
                while ((tconn = fr_heap_peek(trunk->active))) fr_connection_signal_reconnect(tconn->pub.conn, reason);
        }
 
+       RECONNECT_BY_STATE(FR_TRUNK_CONN_FULL, full);
        RECONNECT_BY_STATE(FR_TRUNK_CONN_INACTIVE, inactive);
        RECONNECT_BY_STATE(FR_TRUNK_CONN_FAILED, failed);
        RECONNECT_BY_STATE(FR_TRUNK_CONN_CLOSED, closed);
@@ -3742,6 +3821,7 @@ static int _trunk_free(fr_trunk_t *trunk)
         */
        while ((tconn = fr_heap_peek(trunk->active))) fr_connection_signal_halt(tconn->pub.conn);
        while ((tconn = fr_dlist_head(&trunk->connecting))) fr_connection_signal_halt(tconn->pub.conn);
+       while ((tconn = fr_dlist_head(&trunk->full))) fr_connection_signal_halt(tconn->pub.conn);
        while ((tconn = fr_dlist_head(&trunk->inactive))) fr_connection_signal_halt(tconn->pub.conn);
        while ((tconn = fr_dlist_head(&trunk->failed))) fr_connection_signal_halt(tconn->pub.conn);
        while ((tconn = fr_dlist_head(&trunk->closed))) fr_connection_signal_halt(tconn->pub.conn);
@@ -3830,6 +3910,7 @@ fr_trunk_t *fr_trunk_alloc(TALLOC_CTX *ctx, fr_event_list_t *el,
        MEM(trunk->active = fr_heap_talloc_create(trunk, trunk->funcs.connection_prioritise,
                                                  fr_trunk_connection_t, heap_id));
        fr_dlist_talloc_init(&trunk->connecting, fr_trunk_connection_t, entry);
+       fr_dlist_talloc_init(&trunk->full, fr_trunk_connection_t, entry);
        fr_dlist_talloc_init(&trunk->inactive, fr_trunk_connection_t, entry);
        fr_dlist_talloc_init(&trunk->failed, fr_trunk_connection_t, entry);
        fr_dlist_talloc_init(&trunk->closed, fr_trunk_connection_t, entry);

diff --git a/src/lib/server/trunk.h b/src/lib/server/trunk.h
index c72f4669760714625ed599d1401b68623270f6be..435085685ce3d02b245d641115e0610d4645f735 100644 (file)
--- a/src/lib/server/trunk.h
+++ b/src/lib/server/trunk.h
@@ -82,8 +82,9 @@ typedef enum {
        FR_TRUNK_CONN_FAILED            = 0x04,         //!< Connection failed.  We now wait for it to enter the
                                                        ///< closed state.
        FR_TRUNK_CONN_CLOSED            = 0x08,         //!< Connection was closed, either explicitly or due to failure.
-       FR_TRUNK_CONN_INACTIVE          = 0x10,         //!< Connection is inactive and can't accept any more requests.
-       FR_TRUNK_CONN_DRAINING          = 0x20,         //!< Connection will be closed once it has no more outstanding
+       FR_TRUNK_CONN_FULL              = 0x10,         //!< Connection is full and can't accept any more requests.
+       FR_TRUNK_CONN_INACTIVE          = 0x20,         //!< Connection is inactive and can't accept any more requests.
+       FR_TRUNK_CONN_DRAINING          = 0x30,         //!< Connection will be closed once it has no more outstanding
                                                        ///< requests, if it's not reactivated.
        FR_TRUNK_CONN_DRAINING_TO_FREE  = 0x40          //!< Connection will be closed once it has no more outstanding
                                                        ///< requests.
@@ -98,6 +99,7 @@ typedef enum {
        FR_TRUNK_CONN_ACTIVE | \
        FR_TRUNK_CONN_FAILED | \
        FR_TRUNK_CONN_CLOSED | \
+       FR_TRUNK_CONN_FULL | \
        FR_TRUNK_CONN_INACTIVE | \
        FR_TRUNK_CONN_DRAINING | \
        FR_TRUNK_CONN_DRAINING_TO_FREE \