From: Greg Kroah-Hartman Date: Mon, 11 Apr 2022 17:02:45 +0000 (+0200) Subject: 5.15-stable patches X-Git-Tag: v4.9.310~9 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=5b18cc5f432eee65f68b195afc2c67a011bea924;p=thirdparty%2Fkernel%2Fstable-queue.git 5.15-stable patches added patches: static_call-don-t-make-__static_call_return0-static.patch --- diff --git a/queue-5.15/series b/queue-5.15/series index 77f69453995..d63edfd60da 100644 --- a/queue-5.15/series +++ b/queue-5.15/series @@ -273,3 +273,4 @@ irqchip-gic-v4-wait-for-gicr_vpendbaser.dirty-to-clear-before-descheduling.patch powerpc-64-fix-build-failure-with-allyesconfig-in-book3s_64_entry.s.patch irqchip-gic-gic-v3-prevent-gsi-to-sgi-translations.patch mm-sparsemem-fix-mem_section-will-never-be-null-gcc-12-warning.patch +static_call-don-t-make-__static_call_return0-static.patch diff --git a/queue-5.15/static_call-don-t-make-__static_call_return0-static.patch b/queue-5.15/static_call-don-t-make-__static_call_return0-static.patch new file mode 100644 index 00000000000..a2092e013b9 --- /dev/null +++ b/queue-5.15/static_call-don-t-make-__static_call_return0-static.patch @@ -0,0 +1,1167 @@ +From 8fd4ddda2f49a66bf5dd3d0c01966c4b1971308b Mon Sep 17 00:00:00 2001 +From: Christophe Leroy +Date: Mon, 14 Mar 2022 12:49:36 +0100 +Subject: static_call: Don't make __static_call_return0 static + +From: Christophe Leroy + +commit 8fd4ddda2f49a66bf5dd3d0c01966c4b1971308b upstream. + +System.map shows that vmlinux contains several instances of +__static_call_return0(): + + c0004fc0 t __static_call_return0 + c0011518 t __static_call_return0 + c00d8160 t __static_call_return0 + +arch_static_call_transform() uses the middle one to check whether we are +setting a call to __static_call_return0 or not: + + c0011520 : + c0011520: 3d 20 c0 01 lis r9,-16383 <== r9 = 0xc001 << 16 + c0011524: 39 29 15 18 addi r9,r9,5400 <== r9 += 0x1518 + c0011528: 7c 05 48 00 cmpw r5,r9 <== r9 has value 0xc0011518 here + +So if static_call_update() is called with one of the other instances of +__static_call_return0(), arch_static_call_transform() won't recognise it. + +In order to work properly, global single instance of __static_call_return0() is required. + +Fixes: 3f2a8fc4b15d ("static_call/x86: Add __static_call_return0()") +Signed-off-by: Christophe Leroy +Signed-off-by: Peter Zijlstra (Intel) +Acked-by: Josh Poimboeuf +Link: https://lkml.kernel.org/r/30821468a0e7d28251954b578e5051dc09300d04.1647258493.git.christophe.leroy@csgroup.eu +Signed-off-by: Greg Kroah-Hartman +--- + include/linux/static_call.h | 5 + kernel/Makefile | 3 + kernel/static_call.c | 542 ------------------------------------------- + kernel/static_call_inline.c | 543 ++++++++++++++++++++++++++++++++++++++++++++ + 4 files changed, 547 insertions(+), 546 deletions(-) + create mode 100644 kernel/static_call_inline.c + +--- a/include/linux/static_call.h ++++ b/include/linux/static_call.h +@@ -248,10 +248,7 @@ static inline int static_call_text_reser + return 0; + } + +-static inline long __static_call_return0(void) +-{ +- return 0; +-} ++extern long __static_call_return0(void); + + #define EXPORT_STATIC_CALL(name) \ + EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ +--- a/kernel/Makefile ++++ b/kernel/Makefile +@@ -113,7 +113,8 @@ obj-$(CONFIG_CPU_PM) += cpu_pm.o + obj-$(CONFIG_BPF) += bpf/ + obj-$(CONFIG_KCSAN) += kcsan/ + obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o +-obj-$(CONFIG_HAVE_STATIC_CALL_INLINE) += static_call.o ++obj-$(CONFIG_HAVE_STATIC_CALL) += static_call.o ++obj-$(CONFIG_HAVE_STATIC_CALL_INLINE) += static_call_inline.o + obj-$(CONFIG_CFI_CLANG) += cfi.o + + obj-$(CONFIG_PERF_EVENTS) += events/ +--- a/kernel/static_call.c ++++ b/kernel/static_call.c +@@ -1,548 +1,8 @@ + // SPDX-License-Identifier: GPL-2.0 +-#include + #include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-extern struct static_call_site __start_static_call_sites[], +- __stop_static_call_sites[]; +-extern struct static_call_tramp_key __start_static_call_tramp_key[], +- __stop_static_call_tramp_key[]; +- +-static bool static_call_initialized; +- +-/* mutex to protect key modules/sites */ +-static DEFINE_MUTEX(static_call_mutex); +- +-static void static_call_lock(void) +-{ +- mutex_lock(&static_call_mutex); +-} +- +-static void static_call_unlock(void) +-{ +- mutex_unlock(&static_call_mutex); +-} +- +-static inline void *static_call_addr(struct static_call_site *site) +-{ +- return (void *)((long)site->addr + (long)&site->addr); +-} +- +-static inline unsigned long __static_call_key(const struct static_call_site *site) +-{ +- return (long)site->key + (long)&site->key; +-} +- +-static inline struct static_call_key *static_call_key(const struct static_call_site *site) +-{ +- return (void *)(__static_call_key(site) & ~STATIC_CALL_SITE_FLAGS); +-} +- +-/* These assume the key is word-aligned. */ +-static inline bool static_call_is_init(struct static_call_site *site) +-{ +- return __static_call_key(site) & STATIC_CALL_SITE_INIT; +-} +- +-static inline bool static_call_is_tail(struct static_call_site *site) +-{ +- return __static_call_key(site) & STATIC_CALL_SITE_TAIL; +-} +- +-static inline void static_call_set_init(struct static_call_site *site) +-{ +- site->key = (__static_call_key(site) | STATIC_CALL_SITE_INIT) - +- (long)&site->key; +-} +- +-static int static_call_site_cmp(const void *_a, const void *_b) +-{ +- const struct static_call_site *a = _a; +- const struct static_call_site *b = _b; +- const struct static_call_key *key_a = static_call_key(a); +- const struct static_call_key *key_b = static_call_key(b); +- +- if (key_a < key_b) +- return -1; +- +- if (key_a > key_b) +- return 1; +- +- return 0; +-} +- +-static void static_call_site_swap(void *_a, void *_b, int size) +-{ +- long delta = (unsigned long)_a - (unsigned long)_b; +- struct static_call_site *a = _a; +- struct static_call_site *b = _b; +- struct static_call_site tmp = *a; +- +- a->addr = b->addr - delta; +- a->key = b->key - delta; +- +- b->addr = tmp.addr + delta; +- b->key = tmp.key + delta; +-} +- +-static inline void static_call_sort_entries(struct static_call_site *start, +- struct static_call_site *stop) +-{ +- sort(start, stop - start, sizeof(struct static_call_site), +- static_call_site_cmp, static_call_site_swap); +-} +- +-static inline bool static_call_key_has_mods(struct static_call_key *key) +-{ +- return !(key->type & 1); +-} +- +-static inline struct static_call_mod *static_call_key_next(struct static_call_key *key) +-{ +- if (!static_call_key_has_mods(key)) +- return NULL; +- +- return key->mods; +-} +- +-static inline struct static_call_site *static_call_key_sites(struct static_call_key *key) +-{ +- if (static_call_key_has_mods(key)) +- return NULL; +- +- return (struct static_call_site *)(key->type & ~1); +-} +- +-void __static_call_update(struct static_call_key *key, void *tramp, void *func) +-{ +- struct static_call_site *site, *stop; +- struct static_call_mod *site_mod, first; +- +- cpus_read_lock(); +- static_call_lock(); +- +- if (key->func == func) +- goto done; +- +- key->func = func; +- +- arch_static_call_transform(NULL, tramp, func, false); +- +- /* +- * If uninitialized, we'll not update the callsites, but they still +- * point to the trampoline and we just patched that. +- */ +- if (WARN_ON_ONCE(!static_call_initialized)) +- goto done; +- +- first = (struct static_call_mod){ +- .next = static_call_key_next(key), +- .mod = NULL, +- .sites = static_call_key_sites(key), +- }; +- +- for (site_mod = &first; site_mod; site_mod = site_mod->next) { +- bool init = system_state < SYSTEM_RUNNING; +- struct module *mod = site_mod->mod; +- +- if (!site_mod->sites) { +- /* +- * This can happen if the static call key is defined in +- * a module which doesn't use it. +- * +- * It also happens in the has_mods case, where the +- * 'first' entry has no sites associated with it. +- */ +- continue; +- } +- +- stop = __stop_static_call_sites; +- +- if (mod) { +-#ifdef CONFIG_MODULES +- stop = mod->static_call_sites + +- mod->num_static_call_sites; +- init = mod->state == MODULE_STATE_COMING; +-#endif +- } +- +- for (site = site_mod->sites; +- site < stop && static_call_key(site) == key; site++) { +- void *site_addr = static_call_addr(site); +- +- if (!init && static_call_is_init(site)) +- continue; +- +- if (!kernel_text_address((unsigned long)site_addr)) { +- /* +- * This skips patching built-in __exit, which +- * is part of init_section_contains() but is +- * not part of kernel_text_address(). +- * +- * Skipping built-in __exit is fine since it +- * will never be executed. +- */ +- WARN_ONCE(!static_call_is_init(site), +- "can't patch static call site at %pS", +- site_addr); +- continue; +- } +- +- arch_static_call_transform(site_addr, NULL, func, +- static_call_is_tail(site)); +- } +- } +- +-done: +- static_call_unlock(); +- cpus_read_unlock(); +-} +-EXPORT_SYMBOL_GPL(__static_call_update); +- +-static int __static_call_init(struct module *mod, +- struct static_call_site *start, +- struct static_call_site *stop) +-{ +- struct static_call_site *site; +- struct static_call_key *key, *prev_key = NULL; +- struct static_call_mod *site_mod; +- +- if (start == stop) +- return 0; +- +- static_call_sort_entries(start, stop); +- +- for (site = start; site < stop; site++) { +- void *site_addr = static_call_addr(site); +- +- if ((mod && within_module_init((unsigned long)site_addr, mod)) || +- (!mod && init_section_contains(site_addr, 1))) +- static_call_set_init(site); +- +- key = static_call_key(site); +- if (key != prev_key) { +- prev_key = key; +- +- /* +- * For vmlinux (!mod) avoid the allocation by storing +- * the sites pointer in the key itself. Also see +- * __static_call_update()'s @first. +- * +- * This allows architectures (eg. x86) to call +- * static_call_init() before memory allocation works. +- */ +- if (!mod) { +- key->sites = site; +- key->type |= 1; +- goto do_transform; +- } +- +- site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL); +- if (!site_mod) +- return -ENOMEM; +- +- /* +- * When the key has a direct sites pointer, extract +- * that into an explicit struct static_call_mod, so we +- * can have a list of modules. +- */ +- if (static_call_key_sites(key)) { +- site_mod->mod = NULL; +- site_mod->next = NULL; +- site_mod->sites = static_call_key_sites(key); +- +- key->mods = site_mod; +- +- site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL); +- if (!site_mod) +- return -ENOMEM; +- } +- +- site_mod->mod = mod; +- site_mod->sites = site; +- site_mod->next = static_call_key_next(key); +- key->mods = site_mod; +- } +- +-do_transform: +- arch_static_call_transform(site_addr, NULL, key->func, +- static_call_is_tail(site)); +- } +- +- return 0; +-} +- +-static int addr_conflict(struct static_call_site *site, void *start, void *end) +-{ +- unsigned long addr = (unsigned long)static_call_addr(site); +- +- if (addr <= (unsigned long)end && +- addr + CALL_INSN_SIZE > (unsigned long)start) +- return 1; +- +- return 0; +-} +- +-static int __static_call_text_reserved(struct static_call_site *iter_start, +- struct static_call_site *iter_stop, +- void *start, void *end, bool init) +-{ +- struct static_call_site *iter = iter_start; +- +- while (iter < iter_stop) { +- if (init || !static_call_is_init(iter)) { +- if (addr_conflict(iter, start, end)) +- return 1; +- } +- iter++; +- } +- +- return 0; +-} +- +-#ifdef CONFIG_MODULES +- +-static int __static_call_mod_text_reserved(void *start, void *end) +-{ +- struct module *mod; +- int ret; +- +- preempt_disable(); +- mod = __module_text_address((unsigned long)start); +- WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod); +- if (!try_module_get(mod)) +- mod = NULL; +- preempt_enable(); +- +- if (!mod) +- return 0; +- +- ret = __static_call_text_reserved(mod->static_call_sites, +- mod->static_call_sites + mod->num_static_call_sites, +- start, end, mod->state == MODULE_STATE_COMING); +- +- module_put(mod); +- +- return ret; +-} +- +-static unsigned long tramp_key_lookup(unsigned long addr) +-{ +- struct static_call_tramp_key *start = __start_static_call_tramp_key; +- struct static_call_tramp_key *stop = __stop_static_call_tramp_key; +- struct static_call_tramp_key *tramp_key; +- +- for (tramp_key = start; tramp_key != stop; tramp_key++) { +- unsigned long tramp; +- +- tramp = (long)tramp_key->tramp + (long)&tramp_key->tramp; +- if (tramp == addr) +- return (long)tramp_key->key + (long)&tramp_key->key; +- } +- +- return 0; +-} +- +-static int static_call_add_module(struct module *mod) +-{ +- struct static_call_site *start = mod->static_call_sites; +- struct static_call_site *stop = start + mod->num_static_call_sites; +- struct static_call_site *site; +- +- for (site = start; site != stop; site++) { +- unsigned long s_key = __static_call_key(site); +- unsigned long addr = s_key & ~STATIC_CALL_SITE_FLAGS; +- unsigned long key; +- +- /* +- * Is the key is exported, 'addr' points to the key, which +- * means modules are allowed to call static_call_update() on +- * it. +- * +- * Otherwise, the key isn't exported, and 'addr' points to the +- * trampoline so we need to lookup the key. +- * +- * We go through this dance to prevent crazy modules from +- * abusing sensitive static calls. +- */ +- if (!kernel_text_address(addr)) +- continue; +- +- key = tramp_key_lookup(addr); +- if (!key) { +- pr_warn("Failed to fixup __raw_static_call() usage at: %ps\n", +- static_call_addr(site)); +- return -EINVAL; +- } +- +- key |= s_key & STATIC_CALL_SITE_FLAGS; +- site->key = key - (long)&site->key; +- } +- +- return __static_call_init(mod, start, stop); +-} +- +-static void static_call_del_module(struct module *mod) +-{ +- struct static_call_site *start = mod->static_call_sites; +- struct static_call_site *stop = mod->static_call_sites + +- mod->num_static_call_sites; +- struct static_call_key *key, *prev_key = NULL; +- struct static_call_mod *site_mod, **prev; +- struct static_call_site *site; +- +- for (site = start; site < stop; site++) { +- key = static_call_key(site); +- if (key == prev_key) +- continue; +- +- prev_key = key; +- +- for (prev = &key->mods, site_mod = key->mods; +- site_mod && site_mod->mod != mod; +- prev = &site_mod->next, site_mod = site_mod->next) +- ; +- +- if (!site_mod) +- continue; +- +- *prev = site_mod->next; +- kfree(site_mod); +- } +-} +- +-static int static_call_module_notify(struct notifier_block *nb, +- unsigned long val, void *data) +-{ +- struct module *mod = data; +- int ret = 0; +- +- cpus_read_lock(); +- static_call_lock(); +- +- switch (val) { +- case MODULE_STATE_COMING: +- ret = static_call_add_module(mod); +- if (ret) { +- WARN(1, "Failed to allocate memory for static calls"); +- static_call_del_module(mod); +- } +- break; +- case MODULE_STATE_GOING: +- static_call_del_module(mod); +- break; +- } +- +- static_call_unlock(); +- cpus_read_unlock(); +- +- return notifier_from_errno(ret); +-} +- +-static struct notifier_block static_call_module_nb = { +- .notifier_call = static_call_module_notify, +-}; +- +-#else +- +-static inline int __static_call_mod_text_reserved(void *start, void *end) +-{ +- return 0; +-} +- +-#endif /* CONFIG_MODULES */ +- +-int static_call_text_reserved(void *start, void *end) +-{ +- bool init = system_state < SYSTEM_RUNNING; +- int ret = __static_call_text_reserved(__start_static_call_sites, +- __stop_static_call_sites, start, end, init); +- +- if (ret) +- return ret; +- +- return __static_call_mod_text_reserved(start, end); +-} +- +-int __init static_call_init(void) +-{ +- int ret; +- +- if (static_call_initialized) +- return 0; +- +- cpus_read_lock(); +- static_call_lock(); +- ret = __static_call_init(NULL, __start_static_call_sites, +- __stop_static_call_sites); +- static_call_unlock(); +- cpus_read_unlock(); +- +- if (ret) { +- pr_err("Failed to allocate memory for static_call!\n"); +- BUG(); +- } +- +- static_call_initialized = true; +- +-#ifdef CONFIG_MODULES +- register_module_notifier(&static_call_module_nb); +-#endif +- return 0; +-} +-early_initcall(static_call_init); + + long __static_call_return0(void) + { + return 0; + } +- +-#ifdef CONFIG_STATIC_CALL_SELFTEST +- +-static int func_a(int x) +-{ +- return x+1; +-} +- +-static int func_b(int x) +-{ +- return x+2; +-} +- +-DEFINE_STATIC_CALL(sc_selftest, func_a); +- +-static struct static_call_data { +- int (*func)(int); +- int val; +- int expect; +-} static_call_data [] __initdata = { +- { NULL, 2, 3 }, +- { func_b, 2, 4 }, +- { func_a, 2, 3 } +-}; +- +-static int __init test_static_call_init(void) +-{ +- int i; +- +- for (i = 0; i < ARRAY_SIZE(static_call_data); i++ ) { +- struct static_call_data *scd = &static_call_data[i]; +- +- if (scd->func) +- static_call_update(sc_selftest, scd->func); +- +- WARN_ON(static_call(sc_selftest)(scd->val) != scd->expect); +- } +- +- return 0; +-} +-early_initcall(test_static_call_init); +- +-#endif /* CONFIG_STATIC_CALL_SELFTEST */ ++EXPORT_SYMBOL_GPL(__static_call_return0); +--- /dev/null ++++ b/kernel/static_call_inline.c +@@ -0,0 +1,543 @@ ++// SPDX-License-Identifier: GPL-2.0 ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++extern struct static_call_site __start_static_call_sites[], ++ __stop_static_call_sites[]; ++extern struct static_call_tramp_key __start_static_call_tramp_key[], ++ __stop_static_call_tramp_key[]; ++ ++static bool static_call_initialized; ++ ++/* mutex to protect key modules/sites */ ++static DEFINE_MUTEX(static_call_mutex); ++ ++static void static_call_lock(void) ++{ ++ mutex_lock(&static_call_mutex); ++} ++ ++static void static_call_unlock(void) ++{ ++ mutex_unlock(&static_call_mutex); ++} ++ ++static inline void *static_call_addr(struct static_call_site *site) ++{ ++ return (void *)((long)site->addr + (long)&site->addr); ++} ++ ++static inline unsigned long __static_call_key(const struct static_call_site *site) ++{ ++ return (long)site->key + (long)&site->key; ++} ++ ++static inline struct static_call_key *static_call_key(const struct static_call_site *site) ++{ ++ return (void *)(__static_call_key(site) & ~STATIC_CALL_SITE_FLAGS); ++} ++ ++/* These assume the key is word-aligned. */ ++static inline bool static_call_is_init(struct static_call_site *site) ++{ ++ return __static_call_key(site) & STATIC_CALL_SITE_INIT; ++} ++ ++static inline bool static_call_is_tail(struct static_call_site *site) ++{ ++ return __static_call_key(site) & STATIC_CALL_SITE_TAIL; ++} ++ ++static inline void static_call_set_init(struct static_call_site *site) ++{ ++ site->key = (__static_call_key(site) | STATIC_CALL_SITE_INIT) - ++ (long)&site->key; ++} ++ ++static int static_call_site_cmp(const void *_a, const void *_b) ++{ ++ const struct static_call_site *a = _a; ++ const struct static_call_site *b = _b; ++ const struct static_call_key *key_a = static_call_key(a); ++ const struct static_call_key *key_b = static_call_key(b); ++ ++ if (key_a < key_b) ++ return -1; ++ ++ if (key_a > key_b) ++ return 1; ++ ++ return 0; ++} ++ ++static void static_call_site_swap(void *_a, void *_b, int size) ++{ ++ long delta = (unsigned long)_a - (unsigned long)_b; ++ struct static_call_site *a = _a; ++ struct static_call_site *b = _b; ++ struct static_call_site tmp = *a; ++ ++ a->addr = b->addr - delta; ++ a->key = b->key - delta; ++ ++ b->addr = tmp.addr + delta; ++ b->key = tmp.key + delta; ++} ++ ++static inline void static_call_sort_entries(struct static_call_site *start, ++ struct static_call_site *stop) ++{ ++ sort(start, stop - start, sizeof(struct static_call_site), ++ static_call_site_cmp, static_call_site_swap); ++} ++ ++static inline bool static_call_key_has_mods(struct static_call_key *key) ++{ ++ return !(key->type & 1); ++} ++ ++static inline struct static_call_mod *static_call_key_next(struct static_call_key *key) ++{ ++ if (!static_call_key_has_mods(key)) ++ return NULL; ++ ++ return key->mods; ++} ++ ++static inline struct static_call_site *static_call_key_sites(struct static_call_key *key) ++{ ++ if (static_call_key_has_mods(key)) ++ return NULL; ++ ++ return (struct static_call_site *)(key->type & ~1); ++} ++ ++void __static_call_update(struct static_call_key *key, void *tramp, void *func) ++{ ++ struct static_call_site *site, *stop; ++ struct static_call_mod *site_mod, first; ++ ++ cpus_read_lock(); ++ static_call_lock(); ++ ++ if (key->func == func) ++ goto done; ++ ++ key->func = func; ++ ++ arch_static_call_transform(NULL, tramp, func, false); ++ ++ /* ++ * If uninitialized, we'll not update the callsites, but they still ++ * point to the trampoline and we just patched that. ++ */ ++ if (WARN_ON_ONCE(!static_call_initialized)) ++ goto done; ++ ++ first = (struct static_call_mod){ ++ .next = static_call_key_next(key), ++ .mod = NULL, ++ .sites = static_call_key_sites(key), ++ }; ++ ++ for (site_mod = &first; site_mod; site_mod = site_mod->next) { ++ bool init = system_state < SYSTEM_RUNNING; ++ struct module *mod = site_mod->mod; ++ ++ if (!site_mod->sites) { ++ /* ++ * This can happen if the static call key is defined in ++ * a module which doesn't use it. ++ * ++ * It also happens in the has_mods case, where the ++ * 'first' entry has no sites associated with it. ++ */ ++ continue; ++ } ++ ++ stop = __stop_static_call_sites; ++ ++ if (mod) { ++#ifdef CONFIG_MODULES ++ stop = mod->static_call_sites + ++ mod->num_static_call_sites; ++ init = mod->state == MODULE_STATE_COMING; ++#endif ++ } ++ ++ for (site = site_mod->sites; ++ site < stop && static_call_key(site) == key; site++) { ++ void *site_addr = static_call_addr(site); ++ ++ if (!init && static_call_is_init(site)) ++ continue; ++ ++ if (!kernel_text_address((unsigned long)site_addr)) { ++ /* ++ * This skips patching built-in __exit, which ++ * is part of init_section_contains() but is ++ * not part of kernel_text_address(). ++ * ++ * Skipping built-in __exit is fine since it ++ * will never be executed. ++ */ ++ WARN_ONCE(!static_call_is_init(site), ++ "can't patch static call site at %pS", ++ site_addr); ++ continue; ++ } ++ ++ arch_static_call_transform(site_addr, NULL, func, ++ static_call_is_tail(site)); ++ } ++ } ++ ++done: ++ static_call_unlock(); ++ cpus_read_unlock(); ++} ++EXPORT_SYMBOL_GPL(__static_call_update); ++ ++static int __static_call_init(struct module *mod, ++ struct static_call_site *start, ++ struct static_call_site *stop) ++{ ++ struct static_call_site *site; ++ struct static_call_key *key, *prev_key = NULL; ++ struct static_call_mod *site_mod; ++ ++ if (start == stop) ++ return 0; ++ ++ static_call_sort_entries(start, stop); ++ ++ for (site = start; site < stop; site++) { ++ void *site_addr = static_call_addr(site); ++ ++ if ((mod && within_module_init((unsigned long)site_addr, mod)) || ++ (!mod && init_section_contains(site_addr, 1))) ++ static_call_set_init(site); ++ ++ key = static_call_key(site); ++ if (key != prev_key) { ++ prev_key = key; ++ ++ /* ++ * For vmlinux (!mod) avoid the allocation by storing ++ * the sites pointer in the key itself. Also see ++ * __static_call_update()'s @first. ++ * ++ * This allows architectures (eg. x86) to call ++ * static_call_init() before memory allocation works. ++ */ ++ if (!mod) { ++ key->sites = site; ++ key->type |= 1; ++ goto do_transform; ++ } ++ ++ site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL); ++ if (!site_mod) ++ return -ENOMEM; ++ ++ /* ++ * When the key has a direct sites pointer, extract ++ * that into an explicit struct static_call_mod, so we ++ * can have a list of modules. ++ */ ++ if (static_call_key_sites(key)) { ++ site_mod->mod = NULL; ++ site_mod->next = NULL; ++ site_mod->sites = static_call_key_sites(key); ++ ++ key->mods = site_mod; ++ ++ site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL); ++ if (!site_mod) ++ return -ENOMEM; ++ } ++ ++ site_mod->mod = mod; ++ site_mod->sites = site; ++ site_mod->next = static_call_key_next(key); ++ key->mods = site_mod; ++ } ++ ++do_transform: ++ arch_static_call_transform(site_addr, NULL, key->func, ++ static_call_is_tail(site)); ++ } ++ ++ return 0; ++} ++ ++static int addr_conflict(struct static_call_site *site, void *start, void *end) ++{ ++ unsigned long addr = (unsigned long)static_call_addr(site); ++ ++ if (addr <= (unsigned long)end && ++ addr + CALL_INSN_SIZE > (unsigned long)start) ++ return 1; ++ ++ return 0; ++} ++ ++static int __static_call_text_reserved(struct static_call_site *iter_start, ++ struct static_call_site *iter_stop, ++ void *start, void *end, bool init) ++{ ++ struct static_call_site *iter = iter_start; ++ ++ while (iter < iter_stop) { ++ if (init || !static_call_is_init(iter)) { ++ if (addr_conflict(iter, start, end)) ++ return 1; ++ } ++ iter++; ++ } ++ ++ return 0; ++} ++ ++#ifdef CONFIG_MODULES ++ ++static int __static_call_mod_text_reserved(void *start, void *end) ++{ ++ struct module *mod; ++ int ret; ++ ++ preempt_disable(); ++ mod = __module_text_address((unsigned long)start); ++ WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod); ++ if (!try_module_get(mod)) ++ mod = NULL; ++ preempt_enable(); ++ ++ if (!mod) ++ return 0; ++ ++ ret = __static_call_text_reserved(mod->static_call_sites, ++ mod->static_call_sites + mod->num_static_call_sites, ++ start, end, mod->state == MODULE_STATE_COMING); ++ ++ module_put(mod); ++ ++ return ret; ++} ++ ++static unsigned long tramp_key_lookup(unsigned long addr) ++{ ++ struct static_call_tramp_key *start = __start_static_call_tramp_key; ++ struct static_call_tramp_key *stop = __stop_static_call_tramp_key; ++ struct static_call_tramp_key *tramp_key; ++ ++ for (tramp_key = start; tramp_key != stop; tramp_key++) { ++ unsigned long tramp; ++ ++ tramp = (long)tramp_key->tramp + (long)&tramp_key->tramp; ++ if (tramp == addr) ++ return (long)tramp_key->key + (long)&tramp_key->key; ++ } ++ ++ return 0; ++} ++ ++static int static_call_add_module(struct module *mod) ++{ ++ struct static_call_site *start = mod->static_call_sites; ++ struct static_call_site *stop = start + mod->num_static_call_sites; ++ struct static_call_site *site; ++ ++ for (site = start; site != stop; site++) { ++ unsigned long s_key = __static_call_key(site); ++ unsigned long addr = s_key & ~STATIC_CALL_SITE_FLAGS; ++ unsigned long key; ++ ++ /* ++ * Is the key is exported, 'addr' points to the key, which ++ * means modules are allowed to call static_call_update() on ++ * it. ++ * ++ * Otherwise, the key isn't exported, and 'addr' points to the ++ * trampoline so we need to lookup the key. ++ * ++ * We go through this dance to prevent crazy modules from ++ * abusing sensitive static calls. ++ */ ++ if (!kernel_text_address(addr)) ++ continue; ++ ++ key = tramp_key_lookup(addr); ++ if (!key) { ++ pr_warn("Failed to fixup __raw_static_call() usage at: %ps\n", ++ static_call_addr(site)); ++ return -EINVAL; ++ } ++ ++ key |= s_key & STATIC_CALL_SITE_FLAGS; ++ site->key = key - (long)&site->key; ++ } ++ ++ return __static_call_init(mod, start, stop); ++} ++ ++static void static_call_del_module(struct module *mod) ++{ ++ struct static_call_site *start = mod->static_call_sites; ++ struct static_call_site *stop = mod->static_call_sites + ++ mod->num_static_call_sites; ++ struct static_call_key *key, *prev_key = NULL; ++ struct static_call_mod *site_mod, **prev; ++ struct static_call_site *site; ++ ++ for (site = start; site < stop; site++) { ++ key = static_call_key(site); ++ if (key == prev_key) ++ continue; ++ ++ prev_key = key; ++ ++ for (prev = &key->mods, site_mod = key->mods; ++ site_mod && site_mod->mod != mod; ++ prev = &site_mod->next, site_mod = site_mod->next) ++ ; ++ ++ if (!site_mod) ++ continue; ++ ++ *prev = site_mod->next; ++ kfree(site_mod); ++ } ++} ++ ++static int static_call_module_notify(struct notifier_block *nb, ++ unsigned long val, void *data) ++{ ++ struct module *mod = data; ++ int ret = 0; ++ ++ cpus_read_lock(); ++ static_call_lock(); ++ ++ switch (val) { ++ case MODULE_STATE_COMING: ++ ret = static_call_add_module(mod); ++ if (ret) { ++ WARN(1, "Failed to allocate memory for static calls"); ++ static_call_del_module(mod); ++ } ++ break; ++ case MODULE_STATE_GOING: ++ static_call_del_module(mod); ++ break; ++ } ++ ++ static_call_unlock(); ++ cpus_read_unlock(); ++ ++ return notifier_from_errno(ret); ++} ++ ++static struct notifier_block static_call_module_nb = { ++ .notifier_call = static_call_module_notify, ++}; ++ ++#else ++ ++static inline int __static_call_mod_text_reserved(void *start, void *end) ++{ ++ return 0; ++} ++ ++#endif /* CONFIG_MODULES */ ++ ++int static_call_text_reserved(void *start, void *end) ++{ ++ bool init = system_state < SYSTEM_RUNNING; ++ int ret = __static_call_text_reserved(__start_static_call_sites, ++ __stop_static_call_sites, start, end, init); ++ ++ if (ret) ++ return ret; ++ ++ return __static_call_mod_text_reserved(start, end); ++} ++ ++int __init static_call_init(void) ++{ ++ int ret; ++ ++ if (static_call_initialized) ++ return 0; ++ ++ cpus_read_lock(); ++ static_call_lock(); ++ ret = __static_call_init(NULL, __start_static_call_sites, ++ __stop_static_call_sites); ++ static_call_unlock(); ++ cpus_read_unlock(); ++ ++ if (ret) { ++ pr_err("Failed to allocate memory for static_call!\n"); ++ BUG(); ++ } ++ ++ static_call_initialized = true; ++ ++#ifdef CONFIG_MODULES ++ register_module_notifier(&static_call_module_nb); ++#endif ++ return 0; ++} ++early_initcall(static_call_init); ++ ++#ifdef CONFIG_STATIC_CALL_SELFTEST ++ ++static int func_a(int x) ++{ ++ return x+1; ++} ++ ++static int func_b(int x) ++{ ++ return x+2; ++} ++ ++DEFINE_STATIC_CALL(sc_selftest, func_a); ++ ++static struct static_call_data { ++ int (*func)(int); ++ int val; ++ int expect; ++} static_call_data [] __initdata = { ++ { NULL, 2, 3 }, ++ { func_b, 2, 4 }, ++ { func_a, 2, 3 } ++}; ++ ++static int __init test_static_call_init(void) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(static_call_data); i++ ) { ++ struct static_call_data *scd = &static_call_data[i]; ++ ++ if (scd->func) ++ static_call_update(sc_selftest, scd->func); ++ ++ WARN_ON(static_call(sc_selftest)(scd->val) != scd->expect); ++ } ++ ++ return 0; ++} ++early_initcall(test_static_call_init); ++ ++#endif /* CONFIG_STATIC_CALL_SELFTEST */