From: Alan T. DeKok Date: Tue, 31 Aug 2021 19:36:07 +0000 (-0400) Subject: implement retries for sections and modules X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4c91458e6d51aa2015ea9ce3df6d508f9f80f518;p=thirdparty%2Ffreeradius-server.git implement retries for sections and modules if the module doesn't natively support retries, it's just re-run over and over until it succeeds, or the retry limits are hit. --- diff --git a/src/lib/unlang/interpret.c b/src/lib/unlang/interpret.c index 1ff1cb4afe6..cc51ace3e48 100644 --- a/src/lib/unlang/interpret.c +++ b/src/lib/unlang/interpret.c @@ -189,6 +189,8 @@ int unlang_interpret_push(request_t *request, unlang_t const *instruction, return 0; } +static void instruction_timeout_handler(UNUSED fr_event_list_t *el, UNUSED fr_time_t now, void *ctx); + /** Update the current result after each instruction, and after popping each stack frame * * @param[in] request The current request. @@ -249,6 +251,86 @@ unlang_frame_action_t result_calculate(request_t *request, unlang_stack_frame_t return UNLANG_FRAME_ACTION_POP; } + /* + * The instruction says it should be retried from the beginning. + */ + if (instruction->actions.actions[*result] == MOD_ACTION_RETRY) { + unlang_retry_t *retry = frame->retry; + + RDEBUG4("** [%i] %s - action says to retry with", + stack->depth, __FUNCTION__); + + if (*priority < 0) *priority = 0; + + /* + * If this is the first time doing the retry, + * then allocate the structure and set the timer. + */ + if (!retry) { + frame->retry = retry = talloc_zero(stack, unlang_retry_t); + if (!frame->retry) goto fail; + + retry->request = request; + retry->depth = stack->depth; + retry->state = FR_RETRY_CONTINUE; + retry->count = 1; + + /* + * Set a timer which automatically fires + * if there's a timeout. And parent it + * from the retry structure, so that the + * timer is automatically freed when the + * frame is cleaned up. + */ + if (instruction->actions.retry.mrd) { + retry->timeout = fr_time() + instruction->actions.retry.mrd; + + if (fr_event_timer_at(retry, request->el, &retry->ev, retry->timeout, + instruction_timeout_handler, request) < 0) { + RPEDEBUG("Failed inserting event"); + goto fail; + } + } + + } else { + /* + * We've been told to stop doing retries, + * probably from a timeout. + */ + if (retry->state != FR_RETRY_CONTINUE) goto fail; + + /* + * Clamp it at the maximum count. + */ + if (instruction->actions.retry.mrc > 0) { + if (retry->count >= instruction->actions.retry.mrc) { + retry->state = FR_RETRY_MRC; + + REDEBUG("Retries hit max_rtx_count (%d) - returning 'fail'", instruction->actions.retry.mrc); + + fail: + *result = RLM_MODULE_FAIL; + goto finalize; + } + + retry->count++; + } + } + + RINDENT(); + if (instruction->actions.retry.mrc) { + RDEBUG("... retrying (%d/%d)", retry->count, instruction->actions.retry.mrc); + } else { + RDEBUG("... retrying"); + } + REXDENT(); + + talloc_free(frame->state); + frame_state_init(stack, frame); + return UNLANG_FRAME_ACTION_RETRY; + } + +finalize: /* * The array holds a default priority for this return * code. Grab it in preference to any unset priority. @@ -338,6 +420,7 @@ unlang_frame_action_t frame_eval(request_t *request, unlang_stack_frame_t *frame while (frame->instruction) { unlang_t const *instruction = frame->instruction; unlang_action_t ua = UNLANG_ACTION_UNWIND; + unlang_frame_action_t fa; DUMP_STACK; @@ -481,9 +564,22 @@ unlang_frame_action_t frame_eval(request_t *request, unlang_stack_frame_t *frame */ if (*result != RLM_MODULE_NOT_SET) *priority = instruction->actions.actions[*result]; - if (result_calculate(request, frame, result, priority) == UNLANG_FRAME_ACTION_POP) { + fa = result_calculate(request, frame, result, priority); + switch (fa) { + case UNLANG_FRAME_ACTION_POP: return UNLANG_FRAME_ACTION_POP; + + case UNLANG_FRAME_ACTION_RETRY: + if (unlang_ops[instruction->type].debug_braces) { + REXDENT(); + RDEBUG2("} # retrying the same section"); + } + continue; /* with the current frame */ + + default: + break; } + FALL_THROUGH; /* @@ -630,6 +726,7 @@ CC_HINT(hot) rlm_rcode_t unlang_interpret(request_t *request) fr_table_str_by_value(mod_rcode_table, stack->result, ""), stack->priority); frame_next(stack, frame); + /* * Else if we're really done with this frame * print some helpful debug... @@ -646,6 +743,10 @@ CC_HINT(hot) rlm_rcode_t unlang_interpret(request_t *request) RDEBUG4("** [%i] %s - interpret yielding", stack->depth, __FUNCTION__); intp->funcs.yield(request, intp->uctx); return stack->result; + + case UNLANG_FRAME_ACTION_RETRY: /* retry the current frame */ + fa = UNLANG_FRAME_ACTION_NEXT; + continue; } break; } @@ -959,6 +1060,22 @@ void unlang_interpret_signal(request_t *request, fr_state_signal_t action) } } +static void instruction_timeout_handler(UNUSED fr_event_list_t *el, UNUSED fr_time_t now, void *ctx) +{ + unlang_retry_t *retry = talloc_get_type_abort(ctx, unlang_retry_t); + request_t *request = talloc_get_type_abort(retry->request, request_t); + + RDEBUG("retry timeout reached, signalling interpreter to cancel."); + + /* + * Signal all lower frames to exit. + */ + frame_signal(request, FR_SIGNAL_CANCEL, retry->depth); + + retry->state = FR_RETRY_MRD; + unlang_interpret_mark_runnable(request); +} + /** Return the depth of the request's stack * */ diff --git a/src/lib/unlang/unlang_priv.h b/src/lib/unlang/unlang_priv.h index 33bb1927646..00763f67e9a 100644 --- a/src/lib/unlang/unlang_priv.h +++ b/src/lib/unlang/unlang_priv.h @@ -89,6 +89,7 @@ typedef enum { typedef enum { UNLANG_FRAME_ACTION_POP = 1, //!< Pop the current frame, and check the next one further ///< up in the stack for what to do next. + UNLANG_FRAME_ACTION_RETRY, //!< retry the current frame UNLANG_FRAME_ACTION_NEXT, //!< Process the next instruction at this level. UNLANG_FRAME_ACTION_YIELD //!< Temporarily return control back to the caller on the C ///< stack. @@ -217,6 +218,15 @@ typedef struct { size_t frame_state_pool_size; //!< The total size of the pool to alloc. } unlang_op_t; +typedef struct { + request_t *request; + int depth; //!< of this retry structure + fr_retry_state_t state; + fr_time_t timeout; + uint32_t count; + fr_event_timer_t const *ev; +} unlang_retry_t; + /** Our interpreter stack, as distinct from the C stack * * We don't call the modules recursively. Instead we iterate over a list of #unlang_t and @@ -247,6 +257,8 @@ struct unlang_stack_frame_s { */ void *state; + unlang_retry_t *retry; //!< if the frame is being retried. + rlm_rcode_t result; //!< The result from executing the instruction. int priority; //!< Result priority. When we pop this stack frame ///< this priority will be compared with the one of the @@ -375,6 +387,10 @@ static inline void frame_state_init(unlang_stack_t *stack, unlang_stack_frame_t } else if (op->frame_state_size) { MEM(frame->state = _talloc_zero(stack, op->frame_state_size, name)); } + + /* + * Don't change frame->retry, it may be left over from a previous retry. + */ } /** Cleanup any lingering frame state @@ -419,6 +435,15 @@ static inline void frame_pop(unlang_stack_t *stack) frame = &stack->frame[stack->depth]; + /* + * We clean up the retries when we pop the frame, not + * when we do a frame_cleanup(). That's because + * frame_cleanup() is called from the signal handler, and + * we need to keep frame->retry around to ensure that we + * know how to _stop_ the retries after they've hit a timeout. + */ + talloc_free(frame->retry); + frame_cleanup(frame); frame = &stack->frame[--stack->depth];