Christoph Hellwig | 3320648 | 2019-10-28 13:10:35 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 2 | /* |
| 3 | * SBI initialilization and all extension implementation. |
| 4 | * |
| 5 | * Copyright (c) 2020 Western Digital Corporation or its affiliates. |
| 6 | */ |
Christoph Hellwig | 3320648 | 2019-10-28 13:10:35 +0100 | [diff] [blame] | 7 | |
| 8 | #include <linux/init.h> |
| 9 | #include <linux/pm.h> |
| 10 | #include <asm/sbi.h> |
Atish Patra | 1ef46c2 | 2020-03-17 18:11:38 -0700 | [diff] [blame] | 11 | #include <asm/smp.h> |
Christoph Hellwig | 3320648 | 2019-10-28 13:10:35 +0100 | [diff] [blame] | 12 | |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 13 | /* default SBI version is 0.1 */ |
Jisheng Zhang | de31ea4 | 2021-03-30 02:22:51 +0800 | [diff] [blame] | 14 | unsigned long sbi_spec_version __ro_after_init = SBI_SPEC_VERSION_DEFAULT; |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 15 | EXPORT_SYMBOL(sbi_spec_version); |
| 16 | |
Jisheng Zhang | de31ea4 | 2021-03-30 02:22:51 +0800 | [diff] [blame] | 17 | static void (*__sbi_set_timer)(uint64_t stime) __ro_after_init; |
| 18 | static int (*__sbi_send_ipi)(const unsigned long *hart_mask) __ro_after_init; |
Atish Patra | efca139 | 2020-03-17 18:11:37 -0700 | [diff] [blame] | 19 | static int (*__sbi_rfence)(int fid, const unsigned long *hart_mask, |
| 20 | unsigned long start, unsigned long size, |
Jisheng Zhang | de31ea4 | 2021-03-30 02:22:51 +0800 | [diff] [blame] | 21 | unsigned long arg4, unsigned long arg5) __ro_after_init; |
Atish Patra | efca139 | 2020-03-17 18:11:37 -0700 | [diff] [blame] | 22 | |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 23 | struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, |
| 24 | unsigned long arg1, unsigned long arg2, |
| 25 | unsigned long arg3, unsigned long arg4, |
| 26 | unsigned long arg5) |
| 27 | { |
| 28 | struct sbiret ret; |
| 29 | |
| 30 | register uintptr_t a0 asm ("a0") = (uintptr_t)(arg0); |
| 31 | register uintptr_t a1 asm ("a1") = (uintptr_t)(arg1); |
| 32 | register uintptr_t a2 asm ("a2") = (uintptr_t)(arg2); |
| 33 | register uintptr_t a3 asm ("a3") = (uintptr_t)(arg3); |
| 34 | register uintptr_t a4 asm ("a4") = (uintptr_t)(arg4); |
| 35 | register uintptr_t a5 asm ("a5") = (uintptr_t)(arg5); |
| 36 | register uintptr_t a6 asm ("a6") = (uintptr_t)(fid); |
| 37 | register uintptr_t a7 asm ("a7") = (uintptr_t)(ext); |
| 38 | asm volatile ("ecall" |
| 39 | : "+r" (a0), "+r" (a1) |
| 40 | : "r" (a2), "r" (a3), "r" (a4), "r" (a5), "r" (a6), "r" (a7) |
| 41 | : "memory"); |
| 42 | ret.error = a0; |
| 43 | ret.value = a1; |
| 44 | |
| 45 | return ret; |
| 46 | } |
| 47 | EXPORT_SYMBOL(sbi_ecall); |
| 48 | |
Atish Patra | f90b43c | 2020-03-17 18:11:41 -0700 | [diff] [blame] | 49 | int sbi_err_map_linux_errno(int err) |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 50 | { |
| 51 | switch (err) { |
| 52 | case SBI_SUCCESS: |
| 53 | return 0; |
| 54 | case SBI_ERR_DENIED: |
| 55 | return -EPERM; |
| 56 | case SBI_ERR_INVALID_PARAM: |
| 57 | return -EINVAL; |
| 58 | case SBI_ERR_INVALID_ADDRESS: |
| 59 | return -EFAULT; |
| 60 | case SBI_ERR_NOT_SUPPORTED: |
| 61 | case SBI_ERR_FAILURE: |
| 62 | default: |
| 63 | return -ENOTSUPP; |
| 64 | }; |
| 65 | } |
Atish Patra | f90b43c | 2020-03-17 18:11:41 -0700 | [diff] [blame] | 66 | EXPORT_SYMBOL(sbi_err_map_linux_errno); |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 67 | |
Atish Patra | efca139 | 2020-03-17 18:11:37 -0700 | [diff] [blame] | 68 | #ifdef CONFIG_RISCV_SBI_V01 |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 69 | /** |
| 70 | * sbi_console_putchar() - Writes given character to the console device. |
| 71 | * @ch: The data to be written to the console. |
| 72 | * |
| 73 | * Return: None |
| 74 | */ |
| 75 | void sbi_console_putchar(int ch) |
| 76 | { |
| 77 | sbi_ecall(SBI_EXT_0_1_CONSOLE_PUTCHAR, 0, ch, 0, 0, 0, 0, 0); |
| 78 | } |
| 79 | EXPORT_SYMBOL(sbi_console_putchar); |
| 80 | |
| 81 | /** |
| 82 | * sbi_console_getchar() - Reads a byte from console device. |
| 83 | * |
| 84 | * Returns the value read from console. |
| 85 | */ |
| 86 | int sbi_console_getchar(void) |
| 87 | { |
| 88 | struct sbiret ret; |
| 89 | |
| 90 | ret = sbi_ecall(SBI_EXT_0_1_CONSOLE_GETCHAR, 0, 0, 0, 0, 0, 0, 0); |
| 91 | |
| 92 | return ret.error; |
| 93 | } |
| 94 | EXPORT_SYMBOL(sbi_console_getchar); |
| 95 | |
| 96 | /** |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 97 | * sbi_shutdown() - Remove all the harts from executing supervisor code. |
| 98 | * |
| 99 | * Return: None |
| 100 | */ |
| 101 | void sbi_shutdown(void) |
| 102 | { |
| 103 | sbi_ecall(SBI_EXT_0_1_SHUTDOWN, 0, 0, 0, 0, 0, 0, 0); |
| 104 | } |
Kefeng Wang | 72df61d | 2020-04-17 20:12:20 +0800 | [diff] [blame] | 105 | EXPORT_SYMBOL(sbi_shutdown); |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 106 | |
| 107 | /** |
| 108 | * sbi_clear_ipi() - Clear any pending IPIs for the calling hart. |
| 109 | * |
| 110 | * Return: None |
| 111 | */ |
| 112 | void sbi_clear_ipi(void) |
| 113 | { |
| 114 | sbi_ecall(SBI_EXT_0_1_CLEAR_IPI, 0, 0, 0, 0, 0, 0, 0); |
| 115 | } |
Kefeng Wang | 72df61d | 2020-04-17 20:12:20 +0800 | [diff] [blame] | 116 | EXPORT_SYMBOL(sbi_clear_ipi); |
Atish Patra | efca139 | 2020-03-17 18:11:37 -0700 | [diff] [blame] | 117 | |
| 118 | /** |
Nanyong Sun | 56a6c37f | 2021-03-05 19:33:26 +0800 | [diff] [blame] | 119 | * __sbi_set_timer_v01() - Program the timer for next timer event. |
Atish Patra | efca139 | 2020-03-17 18:11:37 -0700 | [diff] [blame] | 120 | * @stime_value: The value after which next timer event should fire. |
| 121 | * |
| 122 | * Return: None |
| 123 | */ |
| 124 | static void __sbi_set_timer_v01(uint64_t stime_value) |
| 125 | { |
| 126 | #if __riscv_xlen == 32 |
| 127 | sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, |
| 128 | stime_value >> 32, 0, 0, 0, 0); |
| 129 | #else |
| 130 | sbi_ecall(SBI_EXT_0_1_SET_TIMER, 0, stime_value, 0, 0, 0, 0, 0); |
| 131 | #endif |
| 132 | } |
| 133 | |
| 134 | static int __sbi_send_ipi_v01(const unsigned long *hart_mask) |
| 135 | { |
| 136 | sbi_ecall(SBI_EXT_0_1_SEND_IPI, 0, (unsigned long)hart_mask, |
| 137 | 0, 0, 0, 0, 0); |
| 138 | return 0; |
| 139 | } |
| 140 | |
| 141 | static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask, |
| 142 | unsigned long start, unsigned long size, |
| 143 | unsigned long arg4, unsigned long arg5) |
| 144 | { |
| 145 | int result = 0; |
| 146 | |
| 147 | /* v0.2 function IDs are equivalent to v0.1 extension IDs */ |
| 148 | switch (fid) { |
| 149 | case SBI_EXT_RFENCE_REMOTE_FENCE_I: |
| 150 | sbi_ecall(SBI_EXT_0_1_REMOTE_FENCE_I, 0, |
| 151 | (unsigned long)hart_mask, 0, 0, 0, 0, 0); |
| 152 | break; |
| 153 | case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: |
| 154 | sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA, 0, |
| 155 | (unsigned long)hart_mask, start, size, |
| 156 | 0, 0, 0); |
| 157 | break; |
| 158 | case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: |
| 159 | sbi_ecall(SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID, 0, |
| 160 | (unsigned long)hart_mask, start, size, |
| 161 | arg4, 0, 0); |
| 162 | break; |
| 163 | default: |
| 164 | pr_err("SBI call [%d]not supported in SBI v0.1\n", fid); |
| 165 | result = -EINVAL; |
| 166 | } |
| 167 | |
| 168 | return result; |
| 169 | } |
Kefeng Wang | 7d0ce3b | 2020-04-17 20:12:22 +0800 | [diff] [blame] | 170 | |
| 171 | static void sbi_set_power_off(void) |
| 172 | { |
| 173 | pm_power_off = sbi_shutdown; |
| 174 | } |
Atish Patra | efca139 | 2020-03-17 18:11:37 -0700 | [diff] [blame] | 175 | #else |
| 176 | static void __sbi_set_timer_v01(uint64_t stime_value) |
| 177 | { |
| 178 | pr_warn("Timer extension is not available in SBI v%lu.%lu\n", |
| 179 | sbi_major_version(), sbi_minor_version()); |
| 180 | } |
| 181 | |
| 182 | static int __sbi_send_ipi_v01(const unsigned long *hart_mask) |
| 183 | { |
| 184 | pr_warn("IPI extension is not available in SBI v%lu.%lu\n", |
| 185 | sbi_major_version(), sbi_minor_version()); |
| 186 | |
| 187 | return 0; |
| 188 | } |
| 189 | |
| 190 | static int __sbi_rfence_v01(int fid, const unsigned long *hart_mask, |
| 191 | unsigned long start, unsigned long size, |
| 192 | unsigned long arg4, unsigned long arg5) |
| 193 | { |
| 194 | pr_warn("remote fence extension is not available in SBI v%lu.%lu\n", |
| 195 | sbi_major_version(), sbi_minor_version()); |
| 196 | |
| 197 | return 0; |
| 198 | } |
Kefeng Wang | 7d0ce3b | 2020-04-17 20:12:22 +0800 | [diff] [blame] | 199 | |
| 200 | static void sbi_set_power_off(void) {} |
Atish Patra | efca139 | 2020-03-17 18:11:37 -0700 | [diff] [blame] | 201 | #endif /* CONFIG_RISCV_SBI_V01 */ |
| 202 | |
Atish Patra | 1ef46c2 | 2020-03-17 18:11:38 -0700 | [diff] [blame] | 203 | static void __sbi_set_timer_v02(uint64_t stime_value) |
| 204 | { |
| 205 | #if __riscv_xlen == 32 |
| 206 | sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, |
| 207 | stime_value >> 32, 0, 0, 0, 0); |
| 208 | #else |
| 209 | sbi_ecall(SBI_EXT_TIME, SBI_EXT_TIME_SET_TIMER, stime_value, 0, |
| 210 | 0, 0, 0, 0); |
| 211 | #endif |
| 212 | } |
| 213 | |
| 214 | static int __sbi_send_ipi_v02(const unsigned long *hart_mask) |
| 215 | { |
| 216 | unsigned long hartid, hmask_val, hbase; |
| 217 | struct cpumask tmask; |
| 218 | struct sbiret ret = {0}; |
| 219 | int result; |
| 220 | |
| 221 | if (!hart_mask || !(*hart_mask)) { |
| 222 | riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask); |
| 223 | hart_mask = cpumask_bits(&tmask); |
| 224 | } |
| 225 | |
| 226 | hmask_val = 0; |
| 227 | hbase = 0; |
| 228 | for_each_set_bit(hartid, hart_mask, NR_CPUS) { |
| 229 | if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) { |
| 230 | ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, |
| 231 | hmask_val, hbase, 0, 0, 0, 0); |
| 232 | if (ret.error) |
| 233 | goto ecall_failed; |
| 234 | hmask_val = 0; |
| 235 | hbase = 0; |
| 236 | } |
| 237 | if (!hmask_val) |
| 238 | hbase = hartid; |
| 239 | hmask_val |= 1UL << (hartid - hbase); |
| 240 | } |
| 241 | |
| 242 | if (hmask_val) { |
| 243 | ret = sbi_ecall(SBI_EXT_IPI, SBI_EXT_IPI_SEND_IPI, |
| 244 | hmask_val, hbase, 0, 0, 0, 0); |
| 245 | if (ret.error) |
| 246 | goto ecall_failed; |
| 247 | } |
| 248 | |
| 249 | return 0; |
| 250 | |
| 251 | ecall_failed: |
| 252 | result = sbi_err_map_linux_errno(ret.error); |
| 253 | pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", |
| 254 | __func__, hbase, hmask_val, result); |
| 255 | return result; |
| 256 | } |
| 257 | |
| 258 | static int __sbi_rfence_v02_call(unsigned long fid, unsigned long hmask_val, |
| 259 | unsigned long hbase, unsigned long start, |
| 260 | unsigned long size, unsigned long arg4, |
| 261 | unsigned long arg5) |
| 262 | { |
| 263 | struct sbiret ret = {0}; |
| 264 | int ext = SBI_EXT_RFENCE; |
| 265 | int result = 0; |
| 266 | |
| 267 | switch (fid) { |
| 268 | case SBI_EXT_RFENCE_REMOTE_FENCE_I: |
| 269 | ret = sbi_ecall(ext, fid, hmask_val, hbase, 0, 0, 0, 0); |
| 270 | break; |
| 271 | case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: |
| 272 | ret = sbi_ecall(ext, fid, hmask_val, hbase, start, |
| 273 | size, 0, 0); |
| 274 | break; |
| 275 | case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: |
| 276 | ret = sbi_ecall(ext, fid, hmask_val, hbase, start, |
| 277 | size, arg4, 0); |
| 278 | break; |
| 279 | |
| 280 | case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA: |
| 281 | ret = sbi_ecall(ext, fid, hmask_val, hbase, start, |
| 282 | size, 0, 0); |
| 283 | break; |
| 284 | case SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID: |
| 285 | ret = sbi_ecall(ext, fid, hmask_val, hbase, start, |
| 286 | size, arg4, 0); |
| 287 | break; |
| 288 | case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA: |
| 289 | ret = sbi_ecall(ext, fid, hmask_val, hbase, start, |
| 290 | size, 0, 0); |
| 291 | break; |
| 292 | case SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID: |
| 293 | ret = sbi_ecall(ext, fid, hmask_val, hbase, start, |
| 294 | size, arg4, 0); |
| 295 | break; |
| 296 | default: |
| 297 | pr_err("unknown function ID [%lu] for SBI extension [%d]\n", |
| 298 | fid, ext); |
| 299 | result = -EINVAL; |
| 300 | } |
| 301 | |
| 302 | if (ret.error) { |
| 303 | result = sbi_err_map_linux_errno(ret.error); |
| 304 | pr_err("%s: hbase = [%lu] hmask = [0x%lx] failed (error [%d])\n", |
| 305 | __func__, hbase, hmask_val, result); |
| 306 | } |
| 307 | |
| 308 | return result; |
| 309 | } |
| 310 | |
| 311 | static int __sbi_rfence_v02(int fid, const unsigned long *hart_mask, |
| 312 | unsigned long start, unsigned long size, |
| 313 | unsigned long arg4, unsigned long arg5) |
| 314 | { |
| 315 | unsigned long hmask_val, hartid, hbase; |
| 316 | struct cpumask tmask; |
| 317 | int result; |
| 318 | |
| 319 | if (!hart_mask || !(*hart_mask)) { |
| 320 | riscv_cpuid_to_hartid_mask(cpu_online_mask, &tmask); |
| 321 | hart_mask = cpumask_bits(&tmask); |
| 322 | } |
| 323 | |
| 324 | hmask_val = 0; |
| 325 | hbase = 0; |
| 326 | for_each_set_bit(hartid, hart_mask, NR_CPUS) { |
| 327 | if (hmask_val && ((hbase + BITS_PER_LONG) <= hartid)) { |
| 328 | result = __sbi_rfence_v02_call(fid, hmask_val, hbase, |
| 329 | start, size, arg4, arg5); |
| 330 | if (result) |
| 331 | return result; |
| 332 | hmask_val = 0; |
| 333 | hbase = 0; |
| 334 | } |
| 335 | if (!hmask_val) |
| 336 | hbase = hartid; |
| 337 | hmask_val |= 1UL << (hartid - hbase); |
| 338 | } |
| 339 | |
| 340 | if (hmask_val) { |
| 341 | result = __sbi_rfence_v02_call(fid, hmask_val, hbase, |
| 342 | start, size, arg4, arg5); |
| 343 | if (result) |
| 344 | return result; |
| 345 | } |
| 346 | |
| 347 | return 0; |
| 348 | } |
| 349 | |
Atish Patra | efca139 | 2020-03-17 18:11:37 -0700 | [diff] [blame] | 350 | /** |
| 351 | * sbi_set_timer() - Program the timer for next timer event. |
| 352 | * @stime_value: The value after which next timer event should fire. |
| 353 | * |
Atish Patra | 4bb87563 | 2021-02-03 21:26:43 -0800 | [diff] [blame] | 354 | * Return: None. |
Atish Patra | efca139 | 2020-03-17 18:11:37 -0700 | [diff] [blame] | 355 | */ |
| 356 | void sbi_set_timer(uint64_t stime_value) |
| 357 | { |
| 358 | __sbi_set_timer(stime_value); |
| 359 | } |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 360 | |
| 361 | /** |
| 362 | * sbi_send_ipi() - Send an IPI to any hart. |
| 363 | * @hart_mask: A cpu mask containing all the target harts. |
| 364 | * |
Atish Patra | 4bb87563 | 2021-02-03 21:26:43 -0800 | [diff] [blame] | 365 | * Return: 0 on success, appropriate linux error code otherwise. |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 366 | */ |
Atish Patra | 4bb87563 | 2021-02-03 21:26:43 -0800 | [diff] [blame] | 367 | int sbi_send_ipi(const unsigned long *hart_mask) |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 368 | { |
Atish Patra | 4bb87563 | 2021-02-03 21:26:43 -0800 | [diff] [blame] | 369 | return __sbi_send_ipi(hart_mask); |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 370 | } |
| 371 | EXPORT_SYMBOL(sbi_send_ipi); |
| 372 | |
| 373 | /** |
| 374 | * sbi_remote_fence_i() - Execute FENCE.I instruction on given remote harts. |
| 375 | * @hart_mask: A cpu mask containing all the target harts. |
| 376 | * |
Atish Patra | 4bb87563 | 2021-02-03 21:26:43 -0800 | [diff] [blame] | 377 | * Return: 0 on success, appropriate linux error code otherwise. |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 378 | */ |
Atish Patra | 4bb87563 | 2021-02-03 21:26:43 -0800 | [diff] [blame] | 379 | int sbi_remote_fence_i(const unsigned long *hart_mask) |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 380 | { |
Atish Patra | 4bb87563 | 2021-02-03 21:26:43 -0800 | [diff] [blame] | 381 | return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I, |
| 382 | hart_mask, 0, 0, 0, 0); |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 383 | } |
| 384 | EXPORT_SYMBOL(sbi_remote_fence_i); |
| 385 | |
| 386 | /** |
| 387 | * sbi_remote_sfence_vma() - Execute SFENCE.VMA instructions on given remote |
| 388 | * harts for the specified virtual address range. |
| 389 | * @hart_mask: A cpu mask containing all the target harts. |
| 390 | * @start: Start of the virtual address |
| 391 | * @size: Total size of the virtual address range. |
| 392 | * |
Atish Patra | 4bb87563 | 2021-02-03 21:26:43 -0800 | [diff] [blame] | 393 | * Return: 0 on success, appropriate linux error code otherwise. |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 394 | */ |
Atish Patra | 4bb87563 | 2021-02-03 21:26:43 -0800 | [diff] [blame] | 395 | int sbi_remote_sfence_vma(const unsigned long *hart_mask, |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 396 | unsigned long start, |
| 397 | unsigned long size) |
| 398 | { |
Atish Patra | 4bb87563 | 2021-02-03 21:26:43 -0800 | [diff] [blame] | 399 | return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA, |
| 400 | hart_mask, start, size, 0, 0); |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 401 | } |
| 402 | EXPORT_SYMBOL(sbi_remote_sfence_vma); |
| 403 | |
| 404 | /** |
| 405 | * sbi_remote_sfence_vma_asid() - Execute SFENCE.VMA instructions on given |
| 406 | * remote harts for a virtual address range belonging to a specific ASID. |
| 407 | * |
| 408 | * @hart_mask: A cpu mask containing all the target harts. |
| 409 | * @start: Start of the virtual address |
| 410 | * @size: Total size of the virtual address range. |
| 411 | * @asid: The value of address space identifier (ASID). |
| 412 | * |
Atish Patra | 4bb87563 | 2021-02-03 21:26:43 -0800 | [diff] [blame] | 413 | * Return: 0 on success, appropriate linux error code otherwise. |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 414 | */ |
Atish Patra | 4bb87563 | 2021-02-03 21:26:43 -0800 | [diff] [blame] | 415 | int sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 416 | unsigned long start, |
| 417 | unsigned long size, |
| 418 | unsigned long asid) |
| 419 | { |
Atish Patra | 4bb87563 | 2021-02-03 21:26:43 -0800 | [diff] [blame] | 420 | return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID, |
| 421 | hart_mask, start, size, asid, 0); |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 422 | } |
| 423 | EXPORT_SYMBOL(sbi_remote_sfence_vma_asid); |
| 424 | |
| 425 | /** |
Atish Patra | 1ef46c2 | 2020-03-17 18:11:38 -0700 | [diff] [blame] | 426 | * sbi_remote_hfence_gvma() - Execute HFENCE.GVMA instructions on given remote |
| 427 | * harts for the specified guest physical address range. |
| 428 | * @hart_mask: A cpu mask containing all the target harts. |
| 429 | * @start: Start of the guest physical address |
| 430 | * @size: Total size of the guest physical address range. |
| 431 | * |
| 432 | * Return: None |
| 433 | */ |
| 434 | int sbi_remote_hfence_gvma(const unsigned long *hart_mask, |
| 435 | unsigned long start, |
| 436 | unsigned long size) |
| 437 | { |
| 438 | return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA, |
| 439 | hart_mask, start, size, 0, 0); |
| 440 | } |
| 441 | EXPORT_SYMBOL_GPL(sbi_remote_hfence_gvma); |
| 442 | |
| 443 | /** |
| 444 | * sbi_remote_hfence_gvma_vmid() - Execute HFENCE.GVMA instructions on given |
| 445 | * remote harts for a guest physical address range belonging to a specific VMID. |
| 446 | * |
| 447 | * @hart_mask: A cpu mask containing all the target harts. |
| 448 | * @start: Start of the guest physical address |
| 449 | * @size: Total size of the guest physical address range. |
| 450 | * @vmid: The value of guest ID (VMID). |
| 451 | * |
| 452 | * Return: 0 if success, Error otherwise. |
| 453 | */ |
| 454 | int sbi_remote_hfence_gvma_vmid(const unsigned long *hart_mask, |
| 455 | unsigned long start, |
| 456 | unsigned long size, |
| 457 | unsigned long vmid) |
| 458 | { |
| 459 | return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_GVMA_VMID, |
| 460 | hart_mask, start, size, vmid, 0); |
| 461 | } |
| 462 | EXPORT_SYMBOL(sbi_remote_hfence_gvma_vmid); |
| 463 | |
| 464 | /** |
| 465 | * sbi_remote_hfence_vvma() - Execute HFENCE.VVMA instructions on given remote |
| 466 | * harts for the current guest virtual address range. |
| 467 | * @hart_mask: A cpu mask containing all the target harts. |
| 468 | * @start: Start of the current guest virtual address |
| 469 | * @size: Total size of the current guest virtual address range. |
| 470 | * |
| 471 | * Return: None |
| 472 | */ |
| 473 | int sbi_remote_hfence_vvma(const unsigned long *hart_mask, |
| 474 | unsigned long start, |
| 475 | unsigned long size) |
| 476 | { |
| 477 | return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA, |
| 478 | hart_mask, start, size, 0, 0); |
| 479 | } |
| 480 | EXPORT_SYMBOL(sbi_remote_hfence_vvma); |
| 481 | |
| 482 | /** |
| 483 | * sbi_remote_hfence_vvma_asid() - Execute HFENCE.VVMA instructions on given |
| 484 | * remote harts for current guest virtual address range belonging to a specific |
| 485 | * ASID. |
| 486 | * |
| 487 | * @hart_mask: A cpu mask containing all the target harts. |
| 488 | * @start: Start of the current guest virtual address |
| 489 | * @size: Total size of the current guest virtual address range. |
| 490 | * @asid: The value of address space identifier (ASID). |
| 491 | * |
| 492 | * Return: None |
| 493 | */ |
| 494 | int sbi_remote_hfence_vvma_asid(const unsigned long *hart_mask, |
| 495 | unsigned long start, |
| 496 | unsigned long size, |
| 497 | unsigned long asid) |
| 498 | { |
| 499 | return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_HFENCE_VVMA_ASID, |
| 500 | hart_mask, start, size, asid, 0); |
| 501 | } |
| 502 | EXPORT_SYMBOL(sbi_remote_hfence_vvma_asid); |
| 503 | |
| 504 | /** |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 505 | * sbi_probe_extension() - Check if an SBI extension ID is supported or not. |
| 506 | * @extid: The extension ID to be probed. |
| 507 | * |
| 508 | * Return: Extension specific nonzero value f yes, -ENOTSUPP otherwise. |
| 509 | */ |
| 510 | int sbi_probe_extension(int extid) |
| 511 | { |
| 512 | struct sbiret ret; |
| 513 | |
| 514 | ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_PROBE_EXT, extid, |
| 515 | 0, 0, 0, 0, 0); |
| 516 | if (!ret.error) |
| 517 | if (ret.value) |
| 518 | return ret.value; |
| 519 | |
| 520 | return -ENOTSUPP; |
| 521 | } |
| 522 | EXPORT_SYMBOL(sbi_probe_extension); |
| 523 | |
| 524 | static long __sbi_base_ecall(int fid) |
| 525 | { |
| 526 | struct sbiret ret; |
| 527 | |
| 528 | ret = sbi_ecall(SBI_EXT_BASE, fid, 0, 0, 0, 0, 0, 0); |
| 529 | if (!ret.error) |
| 530 | return ret.value; |
| 531 | else |
| 532 | return sbi_err_map_linux_errno(ret.error); |
| 533 | } |
| 534 | |
| 535 | static inline long sbi_get_spec_version(void) |
| 536 | { |
| 537 | return __sbi_base_ecall(SBI_EXT_BASE_GET_SPEC_VERSION); |
| 538 | } |
| 539 | |
| 540 | static inline long sbi_get_firmware_id(void) |
| 541 | { |
| 542 | return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_ID); |
| 543 | } |
| 544 | |
| 545 | static inline long sbi_get_firmware_version(void) |
| 546 | { |
| 547 | return __sbi_base_ecall(SBI_EXT_BASE_GET_IMP_VERSION); |
| 548 | } |
| 549 | |
Vincent Chen | 183787c | 2021-03-22 22:26:02 +0800 | [diff] [blame] | 550 | long sbi_get_mvendorid(void) |
| 551 | { |
| 552 | return __sbi_base_ecall(SBI_EXT_BASE_GET_MVENDORID); |
| 553 | } |
| 554 | |
| 555 | long sbi_get_marchid(void) |
| 556 | { |
| 557 | return __sbi_base_ecall(SBI_EXT_BASE_GET_MARCHID); |
| 558 | } |
| 559 | |
| 560 | long sbi_get_mimpid(void) |
| 561 | { |
| 562 | return __sbi_base_ecall(SBI_EXT_BASE_GET_MIMPID); |
| 563 | } |
| 564 | |
Anup Patel | cc7f3f7 | 2020-08-17 18:12:48 +0530 | [diff] [blame] | 565 | static void sbi_send_cpumask_ipi(const struct cpumask *target) |
| 566 | { |
| 567 | struct cpumask hartid_mask; |
| 568 | |
| 569 | riscv_cpuid_to_hartid_mask(target, &hartid_mask); |
| 570 | |
| 571 | sbi_send_ipi(cpumask_bits(&hartid_mask)); |
| 572 | } |
| 573 | |
Jisheng Zhang | 300f62c | 2021-03-30 02:23:54 +0800 | [diff] [blame] | 574 | static const struct riscv_ipi_ops sbi_ipi_ops = { |
Anup Patel | cc7f3f7 | 2020-08-17 18:12:48 +0530 | [diff] [blame] | 575 | .ipi_inject = sbi_send_cpumask_ipi |
| 576 | }; |
Christoph Hellwig | 3320648 | 2019-10-28 13:10:35 +0100 | [diff] [blame] | 577 | |
Kefeng Wang | 641e8cd | 2020-11-26 10:40:38 +0800 | [diff] [blame] | 578 | void __init sbi_init(void) |
Christoph Hellwig | 3320648 | 2019-10-28 13:10:35 +0100 | [diff] [blame] | 579 | { |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 580 | int ret; |
| 581 | |
Kefeng Wang | 7d0ce3b | 2020-04-17 20:12:22 +0800 | [diff] [blame] | 582 | sbi_set_power_off(); |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 583 | ret = sbi_get_spec_version(); |
| 584 | if (ret > 0) |
| 585 | sbi_spec_version = ret; |
| 586 | |
| 587 | pr_info("SBI specification v%lu.%lu detected\n", |
| 588 | sbi_major_version(), sbi_minor_version()); |
Atish Patra | efca139 | 2020-03-17 18:11:37 -0700 | [diff] [blame] | 589 | |
| 590 | if (!sbi_spec_is_0_1()) { |
Atish Patra | b9dcd9e | 2020-03-17 18:11:35 -0700 | [diff] [blame] | 591 | pr_info("SBI implementation ID=0x%lx Version=0x%lx\n", |
| 592 | sbi_get_firmware_id(), sbi_get_firmware_version()); |
Atish Patra | 1ef46c2 | 2020-03-17 18:11:38 -0700 | [diff] [blame] | 593 | if (sbi_probe_extension(SBI_EXT_TIME) > 0) { |
| 594 | __sbi_set_timer = __sbi_set_timer_v02; |
Anup Patel | f35bb4b | 2021-03-15 16:34:59 +0530 | [diff] [blame] | 595 | pr_info("SBI TIME extension detected\n"); |
Atish Patra | 1ef46c2 | 2020-03-17 18:11:38 -0700 | [diff] [blame] | 596 | } else { |
| 597 | __sbi_set_timer = __sbi_set_timer_v01; |
| 598 | } |
| 599 | if (sbi_probe_extension(SBI_EXT_IPI) > 0) { |
| 600 | __sbi_send_ipi = __sbi_send_ipi_v02; |
Anup Patel | f35bb4b | 2021-03-15 16:34:59 +0530 | [diff] [blame] | 601 | pr_info("SBI IPI extension detected\n"); |
Atish Patra | 1ef46c2 | 2020-03-17 18:11:38 -0700 | [diff] [blame] | 602 | } else { |
| 603 | __sbi_send_ipi = __sbi_send_ipi_v01; |
| 604 | } |
| 605 | if (sbi_probe_extension(SBI_EXT_RFENCE) > 0) { |
| 606 | __sbi_rfence = __sbi_rfence_v02; |
Anup Patel | f35bb4b | 2021-03-15 16:34:59 +0530 | [diff] [blame] | 607 | pr_info("SBI RFENCE extension detected\n"); |
Atish Patra | 1ef46c2 | 2020-03-17 18:11:38 -0700 | [diff] [blame] | 608 | } else { |
| 609 | __sbi_rfence = __sbi_rfence_v01; |
| 610 | } |
| 611 | } else { |
| 612 | __sbi_set_timer = __sbi_set_timer_v01; |
| 613 | __sbi_send_ipi = __sbi_send_ipi_v01; |
| 614 | __sbi_rfence = __sbi_rfence_v01; |
Atish Patra | efca139 | 2020-03-17 18:11:37 -0700 | [diff] [blame] | 615 | } |
| 616 | |
Anup Patel | cc7f3f7 | 2020-08-17 18:12:48 +0530 | [diff] [blame] | 617 | riscv_set_ipi_ops(&sbi_ipi_ops); |
Christoph Hellwig | 3320648 | 2019-10-28 13:10:35 +0100 | [diff] [blame] | 618 | } |