| // SPDX-License-Identifier: GPL-2.0 |
| /* |
| * KFENCE reporting. |
| * |
| * Copyright (C) 2020, Google LLC. |
| */ |
| |
| #include <linux/stdarg.h> |
| |
| #include <linux/kernel.h> |
| #include <linux/lockdep.h> |
| #include <linux/math.h> |
| #include <linux/printk.h> |
| #include <linux/sched/debug.h> |
| #include <linux/seq_file.h> |
| #include <linux/sprintf.h> |
| #include <linux/stacktrace.h> |
| #include <linux/string.h> |
| #include <linux/sched/clock.h> |
| #include <trace/events/error_report.h> |
| |
| #include <asm/kfence.h> |
| |
| #include "kfence.h" |
| |
| /* May be overridden by <asm/kfence.h>. */ |
| #ifndef ARCH_FUNC_PREFIX |
| #define ARCH_FUNC_PREFIX "" |
| #endif |
| |
| /* Helper function to either print to a seq_file or to console. */ |
| __printf(2, 3) |
| static void seq_con_printf(struct seq_file *seq, const char *fmt, ...) |
| { |
| va_list args; |
| |
| va_start(args, fmt); |
| if (seq) |
| seq_vprintf(seq, fmt, args); |
| else |
| vprintk(fmt, args); |
| va_end(args); |
| } |
| |
| /* |
| * Get the number of stack entries to skip to get out of MM internals. @type is |
| * optional, and if set to NULL, assumes an allocation or free stack. |
| */ |
| static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries, |
| const enum kfence_error_type *type) |
| { |
| char buf[64]; |
| int skipnr, fallback = 0; |
| |
| if (type) { |
| /* Depending on error type, find different stack entries. */ |
| switch (*type) { |
| case KFENCE_ERROR_UAF: |
| case KFENCE_ERROR_OOB: |
| case KFENCE_ERROR_INVALID: |
| /* |
| * kfence_handle_page_fault() may be called with pt_regs |
| * set to NULL; in that case we'll simply show the full |
| * stack trace. |
| */ |
| return 0; |
| case KFENCE_ERROR_CORRUPTION: |
| case KFENCE_ERROR_INVALID_FREE: |
| break; |
| } |
| } |
| |
| for (skipnr = 0; skipnr < num_entries; skipnr++) { |
| int len = scnprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skipnr]); |
| |
| if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfence_") || |
| str_has_prefix(buf, ARCH_FUNC_PREFIX "__kfence_") || |
| str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmem_cache_free") || |
| !strncmp(buf, ARCH_FUNC_PREFIX "__slab_free", len)) { |
| /* |
| * In case of tail calls from any of the below to any of |
| * the above, optimized by the compiler such that the |
| * stack trace would omit the initial entry point below. |
| */ |
| fallback = skipnr + 1; |
| } |
| |
| /* |
| * The below list should only include the initial entry points |
| * into the slab allocators. Includes the *_bulk() variants by |
| * checking prefixes. |
| */ |
| if (str_has_prefix(buf, ARCH_FUNC_PREFIX "kfree") || |
| str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_free") || |
| str_has_prefix(buf, ARCH_FUNC_PREFIX "__kmalloc") || |
| str_has_prefix(buf, ARCH_FUNC_PREFIX "kmem_cache_alloc")) |
| goto found; |
| } |
| if (fallback < num_entries) |
| return fallback; |
| found: |
| skipnr++; |
| return skipnr < num_entries ? skipnr : 0; |
| } |
| |
| static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadata *meta, |
| bool show_alloc) |
| { |
| const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track; |
| u64 ts_sec = track->ts_nsec; |
| unsigned long rem_nsec = do_div(ts_sec, NSEC_PER_SEC); |
| u64 interval_nsec = local_clock() - track->ts_nsec; |
| unsigned long rem_interval_nsec = do_div(interval_nsec, NSEC_PER_SEC); |
| |
| /* Timestamp matches printk timestamp format. */ |
| seq_con_printf(seq, "%s by task %d on cpu %d at %lu.%06lus (%lu.%06lus ago):\n", |
| show_alloc ? "allocated" : meta->state == KFENCE_OBJECT_RCU_FREEING ? |
| "rcu freeing" : "freed", track->pid, |
| track->cpu, (unsigned long)ts_sec, rem_nsec / 1000, |
| (unsigned long)interval_nsec, rem_interval_nsec / 1000); |
| |
| if (track->num_stack_entries) { |
| /* Skip allocation/free internals stack. */ |
| int i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL); |
| |
| /* stack_trace_seq_print() does not exist; open code our own. */ |
| for (; i < track->num_stack_entries; i++) |
| seq_con_printf(seq, " %pS\n", (void *)track->stack_entries[i]); |
| } else { |
| seq_con_printf(seq, " no %s stack\n", show_alloc ? "allocation" : "deallocation"); |
| } |
| } |
| |
| void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta) |
| { |
| const int size = abs(meta->size); |
| const unsigned long start = meta->addr; |
| const struct kmem_cache *const cache = meta->cache; |
| |
| lockdep_assert_held(&meta->lock); |
| |
| if (meta->state == KFENCE_OBJECT_UNUSED) { |
| seq_con_printf(seq, "kfence-#%td unused\n", meta - kfence_metadata); |
| return; |
| } |
| |
| seq_con_printf(seq, "kfence-#%td: 0x%p-0x%p, size=%d, cache=%s\n\n", |
| meta - kfence_metadata, (void *)start, (void *)(start + size - 1), |
| size, (cache && cache->name) ? cache->name : "<destroyed>"); |
| |
| kfence_print_stack(seq, meta, true); |
| |
| if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING) { |
| seq_con_printf(seq, "\n"); |
| kfence_print_stack(seq, meta, false); |
| } |
| } |
| |
| /* |
| * Show bytes at @addr that are different from the expected canary values, up to |
| * @max_bytes. |
| */ |
| static void print_diff_canary(unsigned long address, size_t bytes_to_show, |
| const struct kfence_metadata *meta) |
| { |
| const unsigned long show_until_addr = address + bytes_to_show; |
| const u8 *cur, *end; |
| |
| /* Do not show contents of object nor read into following guard page. */ |
| end = (const u8 *)(address < meta->addr ? min(show_until_addr, meta->addr) |
| : min(show_until_addr, PAGE_ALIGN(address))); |
| |
| pr_cont("["); |
| for (cur = (const u8 *)address; cur < end; cur++) { |
| if (*cur == KFENCE_CANARY_PATTERN_U8(cur)) |
| pr_cont(" ."); |
| else if (no_hash_pointers) |
| pr_cont(" 0x%02x", *cur); |
| else /* Do not leak kernel memory in non-debug builds. */ |
| pr_cont(" !"); |
| } |
| pr_cont(" ]"); |
| } |
| |
| static const char *get_access_type(bool is_write) |
| { |
| return is_write ? "write" : "read"; |
| } |
| |
| void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs, |
| const struct kfence_metadata *meta, enum kfence_error_type type) |
| { |
| unsigned long stack_entries[KFENCE_STACK_DEPTH] = { 0 }; |
| const ptrdiff_t object_index = meta ? meta - kfence_metadata : -1; |
| int num_stack_entries; |
| int skipnr = 0; |
| |
| if (regs) { |
| num_stack_entries = stack_trace_save_regs(regs, stack_entries, KFENCE_STACK_DEPTH, 0); |
| } else { |
| num_stack_entries = stack_trace_save(stack_entries, KFENCE_STACK_DEPTH, 1); |
| skipnr = get_stack_skipnr(stack_entries, num_stack_entries, &type); |
| } |
| |
| /* Require non-NULL meta, except if KFENCE_ERROR_INVALID. */ |
| if (WARN_ON(type != KFENCE_ERROR_INVALID && !meta)) |
| return; |
| |
| if (meta) |
| lockdep_assert_held(&meta->lock); |
| /* |
| * Because we may generate reports in printk-unfriendly parts of the |
| * kernel, such as scheduler code, the use of printk() could deadlock. |
| * Until such time that all printing code here is safe in all parts of |
| * the kernel, accept the risk, and just get our message out (given the |
| * system might already behave unpredictably due to the memory error). |
| * As such, also disable lockdep to hide warnings, and avoid disabling |
| * lockdep for the rest of the kernel. |
| */ |
| lockdep_off(); |
| |
| pr_err("==================================================================\n"); |
| /* Print report header. */ |
| switch (type) { |
| case KFENCE_ERROR_OOB: { |
| const bool left_of_object = address < meta->addr; |
| |
| pr_err("BUG: KFENCE: out-of-bounds %s in %pS\n\n", get_access_type(is_write), |
| (void *)stack_entries[skipnr]); |
| pr_err("Out-of-bounds %s at 0x%p (%luB %s of kfence-#%td):\n", |
| get_access_type(is_write), (void *)address, |
| left_of_object ? meta->addr - address : address - meta->addr, |
| left_of_object ? "left" : "right", object_index); |
| break; |
| } |
| case KFENCE_ERROR_UAF: |
| pr_err("BUG: KFENCE: use-after-free %s in %pS\n\n", get_access_type(is_write), |
| (void *)stack_entries[skipnr]); |
| pr_err("Use-after-free %s at 0x%p (in kfence-#%td):\n", |
| get_access_type(is_write), (void *)address, object_index); |
| break; |
| case KFENCE_ERROR_CORRUPTION: |
| pr_err("BUG: KFENCE: memory corruption in %pS\n\n", (void *)stack_entries[skipnr]); |
| pr_err("Corrupted memory at 0x%p ", (void *)address); |
| print_diff_canary(address, 16, meta); |
| pr_cont(" (in kfence-#%td):\n", object_index); |
| break; |
| case KFENCE_ERROR_INVALID: |
| pr_err("BUG: KFENCE: invalid %s in %pS\n\n", get_access_type(is_write), |
| (void *)stack_entries[skipnr]); |
| pr_err("Invalid %s at 0x%p:\n", get_access_type(is_write), |
| (void *)address); |
| break; |
| case KFENCE_ERROR_INVALID_FREE: |
| pr_err("BUG: KFENCE: invalid free in %pS\n\n", (void *)stack_entries[skipnr]); |
| pr_err("Invalid free of 0x%p (in kfence-#%td):\n", (void *)address, |
| object_index); |
| break; |
| } |
| |
| /* Print stack trace and object info. */ |
| stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, 0); |
| |
| if (meta) { |
| pr_err("\n"); |
| kfence_print_object(NULL, meta); |
| } |
| |
| /* Print report footer. */ |
| pr_err("\n"); |
| if (no_hash_pointers && regs) |
| show_regs(regs); |
| else |
| dump_stack_print_info(KERN_ERR); |
| trace_error_report_end(ERROR_DETECTOR_KFENCE, address); |
| pr_err("==================================================================\n"); |
| |
| lockdep_on(); |
| |
| check_panic_on_warn("KFENCE"); |
| |
| /* We encountered a memory safety error, taint the kernel! */ |
| add_taint(TAINT_BAD_PAGE, LOCKDEP_STILL_OK); |
| } |
| |
| #ifdef CONFIG_PRINTK |
| static void kfence_to_kp_stack(const struct kfence_track *track, void **kp_stack) |
| { |
| int i, j; |
| |
| i = get_stack_skipnr(track->stack_entries, track->num_stack_entries, NULL); |
| for (j = 0; i < track->num_stack_entries && j < KS_ADDRS_COUNT; ++i, ++j) |
| kp_stack[j] = (void *)track->stack_entries[i]; |
| if (j < KS_ADDRS_COUNT) |
| kp_stack[j] = NULL; |
| } |
| |
| bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab) |
| { |
| struct kfence_metadata *meta = addr_to_metadata((unsigned long)object); |
| unsigned long flags; |
| |
| if (!meta) |
| return false; |
| |
| /* |
| * If state is UNUSED at least show the pointer requested; the rest |
| * would be garbage data. |
| */ |
| kpp->kp_ptr = object; |
| |
| /* Requesting info an a never-used object is almost certainly a bug. */ |
| if (WARN_ON(meta->state == KFENCE_OBJECT_UNUSED)) |
| return true; |
| |
| raw_spin_lock_irqsave(&meta->lock, flags); |
| |
| kpp->kp_slab = slab; |
| kpp->kp_slab_cache = meta->cache; |
| kpp->kp_objp = (void *)meta->addr; |
| kfence_to_kp_stack(&meta->alloc_track, kpp->kp_stack); |
| if (meta->state == KFENCE_OBJECT_FREED || meta->state == KFENCE_OBJECT_RCU_FREEING) |
| kfence_to_kp_stack(&meta->free_track, kpp->kp_free_stack); |
| /* get_stack_skipnr() ensures the first entry is outside allocator. */ |
| kpp->kp_ret = kpp->kp_stack[0]; |
| |
| raw_spin_unlock_irqrestore(&meta->lock, flags); |
| |
| return true; |
| } |
| #endif |