| /* SPDX-License-Identifier: GPL-2.0 */ |
| #ifndef _LINUX_STATIC_CALL_H |
| #define _LINUX_STATIC_CALL_H |
| |
| /* |
| * Static call support |
| * |
| * Static calls use code patching to hard-code function pointers into direct |
| * branch instructions. They give the flexibility of function pointers, but |
| * with improved performance. This is especially important for cases where |
| * retpolines would otherwise be used, as retpolines can significantly impact |
| * performance. |
| * |
| * |
| * API overview: |
| * |
| * DECLARE_STATIC_CALL(name, func); |
| * DEFINE_STATIC_CALL(name, func); |
| * DEFINE_STATIC_CALL_NULL(name, typename); |
| * static_call(name)(args...); |
| * static_call_cond(name)(args...); |
| * static_call_update(name, func); |
| * |
| * Usage example: |
| * |
| * # Start with the following functions (with identical prototypes): |
| * int func_a(int arg1, int arg2); |
| * int func_b(int arg1, int arg2); |
| * |
| * # Define a 'my_name' reference, associated with func_a() by default |
| * DEFINE_STATIC_CALL(my_name, func_a); |
| * |
| * # Call func_a() |
| * static_call(my_name)(arg1, arg2); |
| * |
| * # Update 'my_name' to point to func_b() |
| * static_call_update(my_name, &func_b); |
| * |
| * # Call func_b() |
| * static_call(my_name)(arg1, arg2); |
| * |
| * |
| * Implementation details: |
| * |
| * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL). |
| * Otherwise basic indirect calls are used (with function pointers). |
| * |
| * Each static_call() site calls into a trampoline associated with the name. |
| * The trampoline has a direct branch to the default function. Updates to a |
| * name will modify the trampoline's branch destination. |
| * |
| * If the arch has CONFIG_HAVE_STATIC_CALL_INLINE, then the call sites |
| * themselves will be patched at runtime to call the functions directly, |
| * rather than calling through the trampoline. This requires objtool or a |
| * compiler plugin to detect all the static_call() sites and annotate them |
| * in the .static_call_sites section. |
| * |
| * |
| * Notes on NULL function pointers: |
| * |
| * Static_call()s support NULL functions, with many of the caveats that |
| * regular function pointers have. |
| * |
| * Clearly calling a NULL function pointer is 'BAD', so too for |
| * static_call()s (although when HAVE_STATIC_CALL it might not be immediately |
| * fatal). A NULL static_call can be the result of: |
| * |
| * DECLARE_STATIC_CALL_NULL(my_static_call, void (*)(int)); |
| * |
| * which is equivalent to declaring a NULL function pointer with just a |
| * typename: |
| * |
| * void (*my_func_ptr)(int arg1) = NULL; |
| * |
| * or using static_call_update() with a NULL function. In both cases the |
| * HAVE_STATIC_CALL implementation will patch the trampoline with a RET |
| * instruction, instead of an immediate tail-call JMP. HAVE_STATIC_CALL_INLINE |
| * architectures can patch the trampoline call to a NOP. |
| * |
| * In all cases, any argument evaluation is unconditional. Unlike a regular |
| * conditional function pointer call: |
| * |
| * if (my_func_ptr) |
| * my_func_ptr(arg1) |
| * |
| * where the argument evaludation also depends on the pointer value. |
| * |
| * When calling a static_call that can be NULL, use: |
| * |
| * static_call_cond(name)(arg1); |
| * |
| * which will include the required value tests to avoid NULL-pointer |
| * dereferences. |
| */ |
| |
| #include <linux/types.h> |
| #include <linux/cpu.h> |
| #include <linux/static_call_types.h> |
| |
| #ifdef CONFIG_HAVE_STATIC_CALL |
| #include <asm/static_call.h> |
| |
| /* |
| * Either @site or @tramp can be NULL. |
| */ |
| extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail); |
| |
| #define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name) |
| |
| #else |
| #define STATIC_CALL_TRAMP_ADDR(name) NULL |
| #endif |
| |
| #define static_call_update(name, func) \ |
| ({ \ |
| BUILD_BUG_ON(!__same_type(*(func), STATIC_CALL_TRAMP(name))); \ |
| __static_call_update(&STATIC_CALL_KEY(name), \ |
| STATIC_CALL_TRAMP_ADDR(name), func); \ |
| }) |
| |
| #ifdef CONFIG_HAVE_STATIC_CALL_INLINE |
| |
| extern int __init static_call_init(void); |
| |
| struct static_call_mod { |
| struct static_call_mod *next; |
| struct module *mod; /* for vmlinux, mod == NULL */ |
| struct static_call_site *sites; |
| }; |
| |
| struct static_call_key { |
| void *func; |
| union { |
| /* bit 0: 0 = mods, 1 = sites */ |
| unsigned long type; |
| struct static_call_mod *mods; |
| struct static_call_site *sites; |
| }; |
| }; |
| |
| /* For finding the key associated with a trampoline */ |
| struct static_call_tramp_key { |
| s32 tramp; |
| s32 key; |
| }; |
| |
| extern void __static_call_update(struct static_call_key *key, void *tramp, void *func); |
| extern int static_call_mod_init(struct module *mod); |
| extern int static_call_text_reserved(void *start, void *end); |
| |
| extern long __static_call_return0(void); |
| |
| #define __DEFINE_STATIC_CALL(name, _func, _func_init) \ |
| DECLARE_STATIC_CALL(name, _func); \ |
| struct static_call_key STATIC_CALL_KEY(name) = { \ |
| .func = _func_init, \ |
| .type = 1, \ |
| }; \ |
| ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func_init) |
| |
| #define DEFINE_STATIC_CALL_NULL(name, _func) \ |
| DECLARE_STATIC_CALL(name, _func); \ |
| struct static_call_key STATIC_CALL_KEY(name) = { \ |
| .func = NULL, \ |
| .type = 1, \ |
| }; \ |
| ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) |
| |
| #define static_call_cond(name) (void)__static_call(name) |
| |
| #define EXPORT_STATIC_CALL(name) \ |
| EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ |
| EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) |
| #define EXPORT_STATIC_CALL_GPL(name) \ |
| EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ |
| EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) |
| |
| /* Leave the key unexported, so modules can't change static call targets: */ |
| #define EXPORT_STATIC_CALL_TRAMP(name) \ |
| EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)); \ |
| ARCH_ADD_TRAMP_KEY(name) |
| #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ |
| EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)); \ |
| ARCH_ADD_TRAMP_KEY(name) |
| |
| #elif defined(CONFIG_HAVE_STATIC_CALL) |
| |
| static inline int static_call_init(void) { return 0; } |
| |
| struct static_call_key { |
| void *func; |
| }; |
| |
| #define __DEFINE_STATIC_CALL(name, _func, _func_init) \ |
| DECLARE_STATIC_CALL(name, _func); \ |
| struct static_call_key STATIC_CALL_KEY(name) = { \ |
| .func = _func_init, \ |
| }; \ |
| ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func_init) |
| |
| #define DEFINE_STATIC_CALL_NULL(name, _func) \ |
| DECLARE_STATIC_CALL(name, _func); \ |
| struct static_call_key STATIC_CALL_KEY(name) = { \ |
| .func = NULL, \ |
| }; \ |
| ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) |
| |
| #define static_call_cond(name) (void)__static_call(name) |
| |
| static inline |
| void __static_call_update(struct static_call_key *key, void *tramp, void *func) |
| { |
| cpus_read_lock(); |
| WRITE_ONCE(key->func, func); |
| arch_static_call_transform(NULL, tramp, func, false); |
| cpus_read_unlock(); |
| } |
| |
| static inline int static_call_text_reserved(void *start, void *end) |
| { |
| return 0; |
| } |
| |
| static inline long __static_call_return0(void) |
| { |
| return 0; |
| } |
| |
| #define EXPORT_STATIC_CALL(name) \ |
| EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \ |
| EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) |
| #define EXPORT_STATIC_CALL_GPL(name) \ |
| EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \ |
| EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) |
| |
| /* Leave the key unexported, so modules can't change static call targets: */ |
| #define EXPORT_STATIC_CALL_TRAMP(name) \ |
| EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)) |
| #define EXPORT_STATIC_CALL_TRAMP_GPL(name) \ |
| EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)) |
| |
| #else /* Generic implementation */ |
| |
| static inline int static_call_init(void) { return 0; } |
| |
| struct static_call_key { |
| void *func; |
| }; |
| |
| static inline long __static_call_return0(void) |
| { |
| return 0; |
| } |
| |
| #define __DEFINE_STATIC_CALL(name, _func, _func_init) \ |
| DECLARE_STATIC_CALL(name, _func); \ |
| struct static_call_key STATIC_CALL_KEY(name) = { \ |
| .func = _func_init, \ |
| } |
| |
| #define DEFINE_STATIC_CALL_NULL(name, _func) \ |
| DECLARE_STATIC_CALL(name, _func); \ |
| struct static_call_key STATIC_CALL_KEY(name) = { \ |
| .func = NULL, \ |
| } |
| |
| static inline void __static_call_nop(void) { } |
| |
| /* |
| * This horrific hack takes care of two things: |
| * |
| * - it ensures the compiler will only load the function pointer ONCE, |
| * which avoids a reload race. |
| * |
| * - it ensures the argument evaluation is unconditional, similar |
| * to the HAVE_STATIC_CALL variant. |
| * |
| * Sadly current GCC/Clang (10 for both) do not optimize this properly |
| * and will emit an indirect call for the NULL case :-( |
| */ |
| #define __static_call_cond(name) \ |
| ({ \ |
| void *func = READ_ONCE(STATIC_CALL_KEY(name).func); \ |
| if (!func) \ |
| func = &__static_call_nop; \ |
| (typeof(STATIC_CALL_TRAMP(name))*)func; \ |
| }) |
| |
| #define static_call_cond(name) (void)__static_call_cond(name) |
| |
| static inline |
| void __static_call_update(struct static_call_key *key, void *tramp, void *func) |
| { |
| WRITE_ONCE(key->func, func); |
| } |
| |
| static inline int static_call_text_reserved(void *start, void *end) |
| { |
| return 0; |
| } |
| |
| #define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name)) |
| #define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)) |
| |
| #endif /* CONFIG_HAVE_STATIC_CALL */ |
| |
| #define DEFINE_STATIC_CALL(name, _func) \ |
| __DEFINE_STATIC_CALL(name, _func, _func) |
| |
| #define DEFINE_STATIC_CALL_RET0(name, _func) \ |
| __DEFINE_STATIC_CALL(name, _func, __static_call_return0) |
| |
| #endif /* _LINUX_STATIC_CALL_H */ |