Thomas Gleixner | 2874c5f | 2019-05-27 08:55:01 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 2 | /* |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 3 | * Copyright (C) 2015 Imagination Technologies |
| 4 | * Author: Alex Smith <alex.smith@imgtec.com> |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 5 | */ |
| 6 | |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 7 | #include <linux/binfmts.h> |
| 8 | #include <linux/elf.h> |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 9 | #include <linux/err.h> |
| 10 | #include <linux/init.h> |
Alex Smith | a7f4df4 | 2015-10-21 09:57:44 +0100 | [diff] [blame] | 11 | #include <linux/ioport.h> |
Paul Burton | 0f02cfb | 2018-08-30 11:01:21 -0700 | [diff] [blame] | 12 | #include <linux/kernel.h> |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 13 | #include <linux/mm.h> |
Paul Burton | ea7e048 | 2018-09-25 15:51:26 -0700 | [diff] [blame] | 14 | #include <linux/random.h> |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 15 | #include <linux/sched.h> |
| 16 | #include <linux/slab.h> |
Alex Smith | a7f4df4 | 2015-10-21 09:57:44 +0100 | [diff] [blame] | 17 | #include <linux/timekeeper_internal.h> |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 18 | |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 19 | #include <asm/abi.h> |
Paul Burton | 00578cd | 2017-08-12 21:36:30 -0700 | [diff] [blame] | 20 | #include <asm/mips-cps.h> |
Paul Burton | 0f02cfb | 2018-08-30 11:01:21 -0700 | [diff] [blame] | 21 | #include <asm/page.h> |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 22 | #include <asm/vdso.h> |
Vincenzo Frascino | 24640f2 | 2019-06-21 10:52:46 +0100 | [diff] [blame] | 23 | #include <vdso/helpers.h> |
| 24 | #include <vdso/vsyscall.h> |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 25 | |
| 26 | /* Kernel-provided data used by the VDSO. */ |
Anna-Maria Behnsen | d697a99 | 2024-02-19 16:39:38 +0100 | [diff] [blame] | 27 | static union vdso_data_store mips_vdso_data __page_aligned_data; |
Vincenzo Frascino | 24640f2 | 2019-06-21 10:52:46 +0100 | [diff] [blame] | 28 | struct vdso_data *vdso_data = mips_vdso_data.data; |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 29 | |
| 30 | /* |
Alex Smith | a7f4df4 | 2015-10-21 09:57:44 +0100 | [diff] [blame] | 31 | * Mapping for the VDSO data/GIC pages. The real pages are mapped manually, as |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 32 | * what we map and where within the area they are mapped is determined at |
| 33 | * runtime. |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 34 | */ |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 35 | static struct page *no_pages[] = { NULL }; |
| 36 | static struct vm_special_mapping vdso_vvar_mapping = { |
| 37 | .name = "[vvar]", |
| 38 | .pages = no_pages, |
| 39 | }; |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 40 | |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 41 | static void __init init_vdso_image(struct mips_vdso_image *image) |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 42 | { |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 43 | unsigned long num_pages, i; |
James Hogan | 554af0c | 2016-09-07 13:37:01 +0100 | [diff] [blame] | 44 | unsigned long data_pfn; |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 45 | |
| 46 | BUG_ON(!PAGE_ALIGNED(image->data)); |
| 47 | BUG_ON(!PAGE_ALIGNED(image->size)); |
| 48 | |
| 49 | num_pages = image->size / PAGE_SIZE; |
| 50 | |
James Hogan | 554af0c | 2016-09-07 13:37:01 +0100 | [diff] [blame] | 51 | data_pfn = __phys_to_pfn(__pa_symbol(image->data)); |
| 52 | for (i = 0; i < num_pages; i++) |
| 53 | image->mapping.pages[i] = pfn_to_page(data_pfn + i); |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 54 | } |
| 55 | |
| 56 | static int __init init_vdso(void) |
| 57 | { |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 58 | init_vdso_image(&vdso_image); |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 59 | |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 60 | #ifdef CONFIG_MIPS32_O32 |
| 61 | init_vdso_image(&vdso_image_o32); |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 62 | #endif |
| 63 | |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 64 | #ifdef CONFIG_MIPS32_N32 |
| 65 | init_vdso_image(&vdso_image_n32); |
| 66 | #endif |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 67 | |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 68 | return 0; |
| 69 | } |
David Daney | 1ed8453 | 2010-06-16 15:00:28 -0700 | [diff] [blame] | 70 | subsys_initcall(init_vdso); |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 71 | |
Paul Burton | ea7e048 | 2018-09-25 15:51:26 -0700 | [diff] [blame] | 72 | static unsigned long vdso_base(void) |
| 73 | { |
Yousong Zhou | aebdc6f | 2020-03-24 23:27:51 +0800 | [diff] [blame] | 74 | unsigned long base = STACK_TOP; |
Paul Burton | ea7e048 | 2018-09-25 15:51:26 -0700 | [diff] [blame] | 75 | |
Yousong Zhou | aebdc6f | 2020-03-24 23:27:51 +0800 | [diff] [blame] | 76 | if (IS_ENABLED(CONFIG_MIPS_FP_SUPPORT)) { |
| 77 | /* Skip the delay slot emulation page */ |
| 78 | base += PAGE_SIZE; |
| 79 | } |
Paul Burton | ea7e048 | 2018-09-25 15:51:26 -0700 | [diff] [blame] | 80 | |
| 81 | if (current->flags & PF_RANDOMIZE) { |
Jason A. Donenfeld | 8032bf1 | 2022-10-09 20:44:02 -0600 | [diff] [blame] | 82 | base += get_random_u32_below(VDSO_RANDOMIZE_SIZE); |
Paul Burton | ea7e048 | 2018-09-25 15:51:26 -0700 | [diff] [blame] | 83 | base = PAGE_ALIGN(base); |
| 84 | } |
| 85 | |
| 86 | return base; |
| 87 | } |
| 88 | |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 89 | int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) |
| 90 | { |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 91 | struct mips_vdso_image *image = current->thread.abi->vdso; |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 92 | struct mm_struct *mm = current->mm; |
Florian Fainelli | dfad83c | 2021-03-30 20:22:07 -0700 | [diff] [blame] | 93 | unsigned long gic_size, vvar_size, size, base, data_addr, vdso_addr, gic_pfn, gic_base; |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 94 | struct vm_area_struct *vma; |
| 95 | int ret; |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 96 | |
Michel Lespinasse | d8ed45c | 2020-06-08 21:33:25 -0700 | [diff] [blame] | 97 | if (mmap_write_lock_killable(mm)) |
Michal Hocko | 6904817 | 2016-05-23 16:25:54 -0700 | [diff] [blame] | 98 | return -EINTR; |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 99 | |
Yousong Zhou | aebdc6f | 2020-03-24 23:27:51 +0800 | [diff] [blame] | 100 | if (IS_ENABLED(CONFIG_MIPS_FP_SUPPORT)) { |
| 101 | /* Map delay slot emulation page */ |
| 102 | base = mmap_region(NULL, STACK_TOP, PAGE_SIZE, |
| 103 | VM_READ | VM_EXEC | |
| 104 | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, |
| 105 | 0, NULL); |
| 106 | if (IS_ERR_VALUE(base)) { |
| 107 | ret = base; |
| 108 | goto out; |
| 109 | } |
Paul Burton | 432c6ba | 2016-07-08 11:06:19 +0100 | [diff] [blame] | 110 | } |
| 111 | |
Alex Smith | a7f4df4 | 2015-10-21 09:57:44 +0100 | [diff] [blame] | 112 | /* |
| 113 | * Determine total area size. This includes the VDSO data itself, the |
| 114 | * data page, and the GIC user page if present. Always create a mapping |
| 115 | * for the GIC user area if the GIC is present regardless of whether it |
| 116 | * is the current clocksource, in case it comes into use later on. We |
| 117 | * only map a page even though the total area is 64K, as we only need |
| 118 | * the counter registers at the start. |
| 119 | */ |
Paul Burton | 00578cd | 2017-08-12 21:36:30 -0700 | [diff] [blame] | 120 | gic_size = mips_gic_present() ? PAGE_SIZE : 0; |
Alex Smith | a7f4df4 | 2015-10-21 09:57:44 +0100 | [diff] [blame] | 121 | vvar_size = gic_size + PAGE_SIZE; |
| 122 | size = vvar_size + image->size; |
| 123 | |
Paul Burton | 0f02cfb | 2018-08-30 11:01:21 -0700 | [diff] [blame] | 124 | /* |
| 125 | * Find a region that's large enough for us to perform the |
| 126 | * colour-matching alignment below. |
| 127 | */ |
| 128 | if (cpu_has_dc_aliases) |
| 129 | size += shm_align_mask + 1; |
| 130 | |
Paul Burton | ea7e048 | 2018-09-25 15:51:26 -0700 | [diff] [blame] | 131 | base = get_unmapped_area(NULL, vdso_base(), size, 0, 0); |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 132 | if (IS_ERR_VALUE(base)) { |
| 133 | ret = base; |
| 134 | goto out; |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 135 | } |
| 136 | |
Paul Burton | 0f02cfb | 2018-08-30 11:01:21 -0700 | [diff] [blame] | 137 | /* |
| 138 | * If we suffer from dcache aliasing, ensure that the VDSO data page |
| 139 | * mapping is coloured the same as the kernel's mapping of that memory. |
| 140 | * This ensures that when the kernel updates the VDSO data userland |
| 141 | * will observe it without requiring cache invalidations. |
| 142 | */ |
| 143 | if (cpu_has_dc_aliases) { |
| 144 | base = __ALIGN_MASK(base, shm_align_mask); |
Vincenzo Frascino | 24640f2 | 2019-06-21 10:52:46 +0100 | [diff] [blame] | 145 | base += ((unsigned long)vdso_data - gic_size) & shm_align_mask; |
Paul Burton | 0f02cfb | 2018-08-30 11:01:21 -0700 | [diff] [blame] | 146 | } |
| 147 | |
Alex Smith | a7f4df4 | 2015-10-21 09:57:44 +0100 | [diff] [blame] | 148 | data_addr = base + gic_size; |
| 149 | vdso_addr = data_addr + PAGE_SIZE; |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 150 | |
Alex Smith | a7f4df4 | 2015-10-21 09:57:44 +0100 | [diff] [blame] | 151 | vma = _install_special_mapping(mm, base, vvar_size, |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 152 | VM_READ | VM_MAYREAD, |
| 153 | &vdso_vvar_mapping); |
| 154 | if (IS_ERR(vma)) { |
| 155 | ret = PTR_ERR(vma); |
| 156 | goto out; |
| 157 | } |
| 158 | |
Alex Smith | a7f4df4 | 2015-10-21 09:57:44 +0100 | [diff] [blame] | 159 | /* Map GIC user page. */ |
| 160 | if (gic_size) { |
Florian Fainelli | dfad83c | 2021-03-30 20:22:07 -0700 | [diff] [blame] | 161 | gic_base = (unsigned long)mips_gic_base + MIPS_GIC_USER_OFS; |
Florian Fainelli | 8baa651 | 2022-07-07 13:27:58 -0700 | [diff] [blame] | 162 | gic_pfn = PFN_DOWN(__pa(gic_base)); |
Alex Smith | a7f4df4 | 2015-10-21 09:57:44 +0100 | [diff] [blame] | 163 | |
Paul Burton | 00578cd | 2017-08-12 21:36:30 -0700 | [diff] [blame] | 164 | ret = io_remap_pfn_range(vma, base, gic_pfn, gic_size, |
Thomas Bogendoerfer | 724d554 | 2020-11-13 12:09:49 +0100 | [diff] [blame] | 165 | pgprot_noncached(vma->vm_page_prot)); |
Alex Smith | a7f4df4 | 2015-10-21 09:57:44 +0100 | [diff] [blame] | 166 | if (ret) |
| 167 | goto out; |
| 168 | } |
| 169 | |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 170 | /* Map data page. */ |
Alex Smith | a7f4df4 | 2015-10-21 09:57:44 +0100 | [diff] [blame] | 171 | ret = remap_pfn_range(vma, data_addr, |
Vincenzo Frascino | 24640f2 | 2019-06-21 10:52:46 +0100 | [diff] [blame] | 172 | virt_to_phys(vdso_data) >> PAGE_SHIFT, |
Thomas Bogendoerfer | 724d554 | 2020-11-13 12:09:49 +0100 | [diff] [blame] | 173 | PAGE_SIZE, vma->vm_page_prot); |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 174 | if (ret) |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 175 | goto out; |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 176 | |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 177 | /* Map VDSO image. */ |
| 178 | vma = _install_special_mapping(mm, vdso_addr, image->size, |
| 179 | VM_READ | VM_EXEC | |
| 180 | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, |
| 181 | &image->mapping); |
| 182 | if (IS_ERR(vma)) { |
| 183 | ret = PTR_ERR(vma); |
| 184 | goto out; |
| 185 | } |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 186 | |
Alex Smith | ebb5e78 | 2015-10-21 09:54:38 +0100 | [diff] [blame] | 187 | mm->context.vdso = (void *)vdso_addr; |
| 188 | ret = 0; |
| 189 | |
| 190 | out: |
Michel Lespinasse | d8ed45c | 2020-06-08 21:33:25 -0700 | [diff] [blame] | 191 | mmap_write_unlock(mm); |
David Daney | c52d0d3 | 2010-02-18 16:13:04 -0800 | [diff] [blame] | 192 | return ret; |
| 193 | } |