tracing/fprobe: Avoid kcalloc() in rcu_read_lock section

fprobe_remove_node_in_module() is called under RCU read locked, but
this invokes kcalloc() if there are more than 8 fprobes installed
on the module. Sashiko warns it because kcalloc() can sleep [1].

 [1] https://sashiko.dev/#/patchset/177552432201.853249.5125045538812833325.stgit%40mhiramat.tok.corp.google.com

To fix this issue, expand the batch size to 128 and do not expand
the fprobe_addr_list, but just cancel walking on fprobe_ip_table,
update fgraph/ftrace_ops and retry the loop again.

Link: https://lore.kernel.org/all/177669367206.132053.1493637946869032744.stgit@mhiramat.tok.corp.google.com/
Fixes: 0de4c70d04a4 ("tracing: fprobe: use rhltable for fprobe_ip_table")
Cc: stable@vger.kernel.org
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>

diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c
index 621477ad0947935e369605a90708eae2a805a67f..9b913facfd3637c4ec826b71febafaf583dd7eb5 100644 (file)
--- a/kernel/trace/fprobe.c
+++ b/kernel/trace/fprobe.c
@@ -344,11 +344,10 @@ static bool fprobe_is_ftrace(struct fprobe *fp)
 }
 
 #ifdef CONFIG_MODULES
-static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove,
-                          int reset)
+static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt)
 {
-       ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset);
-       ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, remove, reset);
+       ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0);
+       ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, 1, 0);
 }
 #endif
 #else
@@ -367,10 +366,9 @@ static bool fprobe_is_ftrace(struct fprobe *fp)
 }
 
 #ifdef CONFIG_MODULES
-static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove,
-                          int reset)
+static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt)
 {
-       ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset);
+       ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0);
 }
 #endif
 #endif /* !CONFIG_DYNAMIC_FTRACE_WITH_ARGS && !CONFIG_DYNAMIC_FTRACE_WITH_REGS */
@@ -542,7 +540,7 @@ static void fprobe_graph_remove_ips(unsigned long *addrs, int num)
 
 #ifdef CONFIG_MODULES
 
-#define FPROBE_IPS_BATCH_INIT 8
+#define FPROBE_IPS_BATCH_INIT 128
 /* instruction pointer address list */
 struct fprobe_addr_list {
        int index;
@@ -550,45 +548,24 @@ struct fprobe_addr_list {
        unsigned long *addrs;
 };
 
-static int fprobe_addr_list_add(struct fprobe_addr_list *alist, unsigned long addr)
+static int fprobe_remove_node_in_module(struct module *mod, struct fprobe_hlist_node *node,
+                                        struct fprobe_addr_list *alist)
 {
-       unsigned long *addrs;
-
-       /* Previously we failed to expand the list. */
-       if (alist->index == alist->size)
-               return -ENOSPC;
-
-       alist->addrs[alist->index++] = addr;
-       if (alist->index < alist->size)
+       if (!within_module(node->addr, mod))
                return 0;
 
-       /* Expand the address list */
-       addrs = kcalloc(alist->size * 2, sizeof(*addrs), GFP_KERNEL);
-       if (!addrs)
-               return -ENOMEM;
-
-       memcpy(addrs, alist->addrs, alist->size * sizeof(*addrs));
-       alist->size *= 2;
-       kfree(alist->addrs);
-       alist->addrs = addrs;
+       if (delete_fprobe_node(node))
+               return 0;
+       /* If no address list is available, we can't track this address. */
+       if (!alist->addrs)
+               return 0;
 
+       alist->addrs[alist->index++] = node->addr;
+       if (alist->index == alist->size)
+               return -ENOSPC;
        return 0;
 }
 
-static void fprobe_remove_node_in_module(struct module *mod, struct fprobe_hlist_node *node,
-                                        struct fprobe_addr_list *alist)
-{
-       if (!within_module(node->addr, mod))
-               return;
-       if (delete_fprobe_node(node))
-               return;
-       /*
-        * If failed to update alist, just continue to update hlist.
-        * Therefore, at list user handler will not hit anymore.
-        */
-       fprobe_addr_list_add(alist, node->addr);
-}
-
 /* Handle module unloading to manage fprobe_ip_table. */
 static int fprobe_module_callback(struct notifier_block *nb,
                                  unsigned long val, void *data)
@@ -597,29 +574,50 @@ static int fprobe_module_callback(struct notifier_block *nb,
        struct fprobe_hlist_node *node;
        struct rhashtable_iter iter;
        struct module *mod = data;
+       bool retry;
 
        if (val != MODULE_STATE_GOING)
                return NOTIFY_DONE;
 
        alist.addrs = kcalloc(alist.size, sizeof(*alist.addrs), GFP_KERNEL);
-       /* If failed to alloc memory, we can not remove ips from hash. */
-       if (!alist.addrs)
-               return NOTIFY_DONE;
+       /*
+        * If failed to alloc memory, ftrace_ops will not be able to remove ips from
+        * hash, but we can still remove nodes from fprobe_ip_table, so we can avoid
+        * the potential wrong callback. So just print a warning here and try to
+        * continue without address list.
+        */
+       WARN_ONCE(!alist.addrs,
+               "Failed to allocate memory for fprobe_addr_list, ftrace_ops will not be updated");
 
        mutex_lock(&fprobe_mutex);
+again:
+       retry = false;
+       alist.index = 0;
        rhltable_walk_enter(&fprobe_ip_table, &iter);
        do {
                rhashtable_walk_start(&iter);
 
                while ((node = rhashtable_walk_next(&iter)) && !IS_ERR(node))
-                       fprobe_remove_node_in_module(mod, node, &alist);
+                       if (fprobe_remove_node_in_module(mod, node, &alist) < 0) {
+                               retry = true;
+                               break;
+                       }
 
                rhashtable_walk_stop(&iter);
-       } while (node == ERR_PTR(-EAGAIN));
+       } while (node == ERR_PTR(-EAGAIN) && !retry);
        rhashtable_walk_exit(&iter);
+       /* Remove any ips from hash table(s) */
+       if (alist.index > 0) {
+               fprobe_remove_ips(alist.addrs, alist.index);
+               /*
+                * If we break rhashtable walk loop except for -EAGAIN, we need
+                * to restart looping from start for safety. Anyway, this is
+                * not a hotpath.
+                */
+               if (retry)
+                       goto again;
+       }
 
-       if (alist.index > 0)
-               fprobe_set_ips(alist.addrs, alist.index, 1, 0);
        mutex_unlock(&fprobe_mutex);
 
        kfree(alist.addrs);