Restrict CLI/AMI commands on shutdown.

During stress testing, we have frequently seen crashes occur because a
CLI or AMI command attempts to access information that is in the process
of being destroyed.

When addressing how to fix this issue, we initially considered fixing
individual crashes we observed. However, the changes required to fix
those problems would introduce considerable overhead to the nominal
case. This is not reasonable in order to prevent a crash from occurring
while Asterisk is already shutting down.

Instead, this change makes it so AMI and CLI commands cannot be executed
if Asterisk is being shut down. For AMI, this is absolute. For CLI,
though, certain commands can be registered so that they may be run
during Asterisk shutdown.

ASTERISK-25825 #close

Change-Id: I8887e215ac352fadf7f4c1e082da9089b1421990

diff --git a/include/asterisk/cli.h b/include/asterisk/cli.h
index 0bda6665ce3e6c01b32b5fb33c44b2e3a5670768..c79a4e93ce649eaf7c1921ba5c3fa87b188e79be 100644 (file)
--- a/include/asterisk/cli.h
+++ b/include/asterisk/cli.h
@@ -326,6 +326,18 @@ char *ast_complete_channels(const char *line, const char *word, int pos, int sta
  */
 void ast_cli_print_timestr_fromseconds(int fd, int seconds, const char *prefix);
 
+/*
+ * \brief Allow a CLI command to be executed while Asterisk is shutting down.
+ *
+ * CLI commands by defeault are disabled when Asterisk is shutting down. This is
+ * to ensure the safety of the shutdown since CLI commands may attempt to access
+ * resources that have been freed as a result of the shutdown.
+ *
+ * If a CLI command should be allowed at shutdown, then the best way to enable this
+ * is to call ast_cli_allow_at_shutdown during the CLI_INIT state of the CLI handler.
+ */
+int ast_cli_allow_at_shutdown(struct ast_cli_entry *e);
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }
 #endif

diff --git a/main/asterisk.c b/main/asterisk.c
index da804e19639159d0748604a81e27d19e716b7af7..7636ec7b40ab035fa94828249a948a6ada915e5d 100644 (file)
--- a/main/asterisk.c
+++ b/main/asterisk.c
@@ -2343,6 +2343,7 @@ static char *handle_stop_now(struct ast_cli_entry *e, int cmd, struct ast_cli_ar
                e->usage =
                        "Usage: core stop now\n"
                        "       Shuts down a running Asterisk immediately, hanging up all active calls .\n";
+               ast_cli_allow_at_shutdown(e);
                return NULL;
        case CLI_GENERATE:
                return NULL;
@@ -2363,6 +2364,7 @@ static char *handle_stop_gracefully(struct ast_cli_entry *e, int cmd, struct ast
                        "Usage: core stop gracefully\n"
                        "       Causes Asterisk to not accept new calls, and exit when all\n"
                        "       active calls have terminated normally.\n";
+               ast_cli_allow_at_shutdown(e);
                return NULL;
        case CLI_GENERATE:
                return NULL;
@@ -2382,6 +2384,7 @@ static char *handle_stop_when_convenient(struct ast_cli_entry *e, int cmd, struc
                e->usage =
                        "Usage: core stop when convenient\n"
                        "       Causes Asterisk to perform a shutdown when all active calls have ended.\n";
+               ast_cli_allow_at_shutdown(e);
                return NULL;
        case CLI_GENERATE:
                return NULL;
@@ -2403,6 +2406,7 @@ static char *handle_restart_now(struct ast_cli_entry *e, int cmd, struct ast_cli
                        "Usage: core restart now\n"
                        "       Causes Asterisk to hangup all calls and exec() itself performing a cold\n"
                        "       restart.\n";
+               ast_cli_allow_at_shutdown(e);
                return NULL;
        case CLI_GENERATE:
                return NULL;
@@ -2423,6 +2427,7 @@ static char *handle_restart_gracefully(struct ast_cli_entry *e, int cmd, struct
                        "Usage: core restart gracefully\n"
                        "       Causes Asterisk to stop accepting new calls and exec() itself performing a cold\n"
                        "       restart when all active calls have ended.\n";
+               ast_cli_allow_at_shutdown(e);
                return NULL;
        case CLI_GENERATE:
                return NULL;
@@ -2442,6 +2447,7 @@ static char *handle_restart_when_convenient(struct ast_cli_entry *e, int cmd, st
                e->usage =
                        "Usage: core restart when convenient\n"
                        "       Causes Asterisk to perform a cold restart when all active calls have ended.\n";
+               ast_cli_allow_at_shutdown(e);
                return NULL;
        case CLI_GENERATE:
                return NULL;
@@ -2463,6 +2469,7 @@ static char *handle_abort_shutdown(struct ast_cli_entry *e, int cmd, struct ast_
                        "Usage: core abort shutdown\n"
                        "       Causes Asterisk to abort an executing shutdown or restart, and resume normal\n"
                        "       call operations.\n";
+               ast_cli_allow_at_shutdown(e);
                return NULL;
        case CLI_GENERATE:
                return NULL;

diff --git a/main/cli.c b/main/cli.c
index 0ac5d612a0af7c01f13e7af23e6ad8b69cd24da3..f2bedc91abaf7b904a121bf11c468a488ecd1eb8 100644 (file)
--- a/main/cli.c
+++ b/main/cli.c
@@ -63,6 +63,7 @@ ASTERISK_REGISTER_FILE()
 #include "asterisk/bridge.h"
 #include "asterisk/stasis_channels.h"
 #include "asterisk/stasis_bridges.h"
+#include "asterisk/vector.h"
 
 /*!
  * \brief List of restrictions per user.
@@ -109,6 +110,9 @@ static struct module_level_list debug_modules = AST_RWLIST_HEAD_INIT_VALUE;
 
 AST_THREADSTORAGE(ast_cli_buf);
 
+AST_RWLOCK_DEFINE_STATIC(shutdown_commands_lock);
+static AST_VECTOR(, struct ast_cli_entry *) shutdown_commands;
+
 /*! \brief Initial buffer size for resulting strings in ast_cli() */
 #define AST_CLI_INITLEN   256
 
@@ -2031,6 +2035,7 @@ static void cli_shutdown(void)
 /*! \brief initialize the _full_cmd string in * each of the builtins. */
 void ast_builtins_init(void)
 {
+       AST_VECTOR_INIT(&shutdown_commands, 0);
        ast_cli_register_multiple(cli_cli, ARRAY_LEN(cli_cli));
        ast_register_cleanup(cli_shutdown);
 }
@@ -2209,6 +2214,13 @@ static int cli_is_registered(struct ast_cli_entry *e)
        return 0;
 }
 
+static void remove_shutdown_command(struct ast_cli_entry *e)
+{
+       ast_rwlock_wrlock(&shutdown_commands_lock);
+       AST_VECTOR_REMOVE_ELEM_UNORDERED(&shutdown_commands, e, AST_VECTOR_ELEM_CLEANUP_NOOP);
+       ast_rwlock_unlock(&shutdown_commands_lock);
+}
+
 int ast_cli_unregister(struct ast_cli_entry *e)
 {
        if (e->inuse) {
@@ -2217,6 +2229,7 @@ int ast_cli_unregister(struct ast_cli_entry *e)
                AST_RWLIST_WRLOCK(&helpers);
                AST_RWLIST_REMOVE(&helpers, e, list);
                AST_RWLIST_UNLOCK(&helpers);
+               remove_shutdown_command(e);
                ast_free(e->_full_cmd);
                e->_full_cmd = NULL;
                if (e->handler) {
@@ -2675,10 +2688,27 @@ char *ast_cli_generator(const char *text, const char *word, int state)
        return __ast_cli_generator(text, word, state, 1);
 }
 
+static int allowed_on_shutdown(struct ast_cli_entry *e)
+{
+       int found = 0;
+       int i;
+
+       ast_rwlock_rdlock(&shutdown_commands_lock);
+       for (i = 0; i < AST_VECTOR_SIZE(&shutdown_commands); ++i) {
+               if (e == AST_VECTOR_GET(&shutdown_commands, i)) {
+                       found = 1;
+                       break;
+               }
+       }
+       ast_rwlock_unlock(&shutdown_commands_lock);
+
+       return found;
+}
+
 int ast_cli_command_full(int uid, int gid, int fd, const char *s)
 {
        const char *args[AST_MAX_ARGS + 1];
-       struct ast_cli_entry *e;
+       struct ast_cli_entry *e = NULL;
        int x;
        char *duplicate = parse_args(s, &x, args + 1, AST_MAX_ARGS, NULL);
        char tmp[AST_MAX_ARGS + 1];
@@ -2702,6 +2732,11 @@ int ast_cli_command_full(int uid, int gid, int fd, const char *s)
                goto done;
        }
 
+       if (ast_shutting_down() && !allowed_on_shutdown(e)) {
+               ast_cli(fd, "Command '%s' cannot be run during shutdown\n", s);
+               goto done;
+       }
+
        ast_join(tmp, sizeof(tmp), args + 1);
        /* Check if the user has rights to run this command. */
        if (!cli_has_permissions(uid, gid, tmp)) {
@@ -2724,8 +2759,11 @@ int ast_cli_command_full(int uid, int gid, int fd, const char *s)
        } else if (retval == CLI_FAILURE) {
                ast_cli(fd, "Command '%s' failed.\n", s);
        }
-       ast_atomic_fetchadd_int(&e->inuse, -1);
+
 done:
+       if (e) {
+               ast_atomic_fetchadd_int(&e->inuse, -1);
+       }
        ast_free(duplicate);
        return retval == CLI_SUCCESS ? RESULT_SUCCESS : RESULT_FAILURE;
 }
@@ -2751,3 +2789,14 @@ void ast_cli_print_timestr_fromseconds(int fd, int seconds, const char *prefix)
 {
        print_uptimestr(fd, ast_tv(seconds, 0), prefix, 0);
 }
+
+int ast_cli_allow_at_shutdown(struct ast_cli_entry *e)
+{
+       int res;
+
+       ast_rwlock_wrlock(&shutdown_commands_lock);
+       res = AST_VECTOR_APPEND(&shutdown_commands, e);
+       ast_rwlock_unlock(&shutdown_commands_lock);
+
+       return res;
+}

diff --git a/main/manager.c b/main/manager.c
index 2adcb3e5a6a96d9742f8d5b59fd8b6edeb91f2f4..e74b253ff295e4cf8ca810a0803c71cafe12c88c 100644 (file)
--- a/main/manager.c
+++ b/main/manager.c
@@ -6138,6 +6138,14 @@ static int process_message(struct mansession *s, const struct message *m)
                return 0;
        }
 
+       if (ast_shutting_down()) {
+               ast_log(LOG_ERROR, "Unable to process manager action '%s'. Asterisk is shutting down.\n", action);
+               mansession_lock(s);
+               astman_send_error(s, m, "Asterisk is shutting down");
+               mansession_unlock(s);
+               return 0;
+       }
+
        if (!s->session->authenticated
                && strcasecmp(action, "Login")
                && strcasecmp(action, "Logoff")

diff --git a/main/utils.c b/main/utils.c
index 8a9f91062d9f79d039458d7b836e56358e6f0a6f..6a778b90c4ac1decce2c018abd96e0e6f4414b3a 100644 (file)
--- a/main/utils.c
+++ b/main/utils.c
@@ -1153,6 +1153,7 @@ static char *handle_show_locks(struct ast_cli_entry *e, int cmd, struct ast_cli_
                        "Usage: core show locks\n"
                        "       This command is for lock debugging.  It prints out which locks\n"
                        "are owned by each active thread.\n";
+               ast_cli_allow_on_shutdown(e);
                return NULL;
 
        case CLI_GENERATE: