| /* SPDX-License-Identifier: GPL-2.0 */ |
| #ifndef __ASM_ALTERNATIVE_MACROS_H |
| #define __ASM_ALTERNATIVE_MACROS_H |
| |
| #include <linux/const.h> |
| #include <vdso/bits.h> |
| |
| #include <asm/cpucaps.h> |
| #include <asm/insn-def.h> |
| |
| /* |
| * Binutils 2.27.0 can't handle a 'UL' suffix on constants, so for the assembly |
| * macros below we must use we must use `(1 << ARM64_CB_SHIFT)`. |
| */ |
| #define ARM64_CB_SHIFT 15 |
| #define ARM64_CB_BIT BIT(ARM64_CB_SHIFT) |
| |
| #if ARM64_NCAPS >= ARM64_CB_BIT |
| #error "cpucaps have overflown ARM64_CB_BIT" |
| #endif |
| |
| #ifndef __ASSEMBLY__ |
| |
| #include <linux/stringify.h> |
| |
| #define ALTINSTR_ENTRY(feature) \ |
| " .word 661b - .\n" /* label */ \ |
| " .word 663f - .\n" /* new instruction */ \ |
| " .hword " __stringify(feature) "\n" /* feature bit */ \ |
| " .byte 662b-661b\n" /* source len */ \ |
| " .byte 664f-663f\n" /* replacement len */ |
| |
| #define ALTINSTR_ENTRY_CB(feature, cb) \ |
| " .word 661b - .\n" /* label */ \ |
| " .word " __stringify(cb) "- .\n" /* callback */ \ |
| " .hword " __stringify(feature) "\n" /* feature bit */ \ |
| " .byte 662b-661b\n" /* source len */ \ |
| " .byte 664f-663f\n" /* replacement len */ |
| |
| /* |
| * alternative assembly primitive: |
| * |
| * If any of these .org directive fail, it means that insn1 and insn2 |
| * don't have the same length. This used to be written as |
| * |
| * .if ((664b-663b) != (662b-661b)) |
| * .error "Alternatives instruction length mismatch" |
| * .endif |
| * |
| * but most assemblers die if insn1 or insn2 have a .inst. This should |
| * be fixed in a binutils release posterior to 2.25.51.0.2 (anything |
| * containing commit 4e4d08cf7399b606 or c1baaddf8861). |
| * |
| * Alternatives with callbacks do not generate replacement instructions. |
| */ |
| #define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled) \ |
| ".if "__stringify(cfg_enabled)" == 1\n" \ |
| "661:\n\t" \ |
| oldinstr "\n" \ |
| "662:\n" \ |
| ".pushsection .altinstructions,\"a\"\n" \ |
| ALTINSTR_ENTRY(feature) \ |
| ".popsection\n" \ |
| ".subsection 1\n" \ |
| "663:\n\t" \ |
| newinstr "\n" \ |
| "664:\n\t" \ |
| ".org . - (664b-663b) + (662b-661b)\n\t" \ |
| ".org . - (662b-661b) + (664b-663b)\n\t" \ |
| ".previous\n" \ |
| ".endif\n" |
| |
| #define __ALTERNATIVE_CFG_CB(oldinstr, feature, cfg_enabled, cb) \ |
| ".if "__stringify(cfg_enabled)" == 1\n" \ |
| "661:\n\t" \ |
| oldinstr "\n" \ |
| "662:\n" \ |
| ".pushsection .altinstructions,\"a\"\n" \ |
| ALTINSTR_ENTRY_CB(feature, cb) \ |
| ".popsection\n" \ |
| "663:\n\t" \ |
| "664:\n\t" \ |
| ".endif\n" |
| |
| #define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \ |
| __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg)) |
| |
| #define ALTERNATIVE_CB(oldinstr, feature, cb) \ |
| __ALTERNATIVE_CFG_CB(oldinstr, (1 << ARM64_CB_SHIFT) | (feature), 1, cb) |
| #else |
| |
| #include <asm/assembler.h> |
| |
| .macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len |
| .word \orig_offset - . |
| .word \alt_offset - . |
| .hword (\feature) |
| .byte \orig_len |
| .byte \alt_len |
| .endm |
| |
| .macro alternative_insn insn1, insn2, cap, enable = 1 |
| .if \enable |
| 661: \insn1 |
| 662: .pushsection .altinstructions, "a" |
| altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f |
| .popsection |
| .subsection 1 |
| 663: \insn2 |
| 664: .org . - (664b-663b) + (662b-661b) |
| .org . - (662b-661b) + (664b-663b) |
| .previous |
| .endif |
| .endm |
| |
| /* |
| * Alternative sequences |
| * |
| * The code for the case where the capability is not present will be |
| * assembled and linked as normal. There are no restrictions on this |
| * code. |
| * |
| * The code for the case where the capability is present will be |
| * assembled into a special section to be used for dynamic patching. |
| * Code for that case must: |
| * |
| * 1. Be exactly the same length (in bytes) as the default code |
| * sequence. |
| * |
| * 2. Not contain a branch target that is used outside of the |
| * alternative sequence it is defined in (branches into an |
| * alternative sequence are not fixed up). |
| */ |
| |
| /* |
| * Begin an alternative code sequence. |
| */ |
| .macro alternative_if_not cap |
| .set .Lasm_alt_mode, 0 |
| .pushsection .altinstructions, "a" |
| altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f |
| .popsection |
| 661: |
| .endm |
| |
| .macro alternative_if cap |
| .set .Lasm_alt_mode, 1 |
| .pushsection .altinstructions, "a" |
| altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f |
| .popsection |
| .subsection 1 |
| .align 2 /* So GAS knows label 661 is suitably aligned */ |
| 661: |
| .endm |
| |
| .macro alternative_cb cap, cb |
| .set .Lasm_alt_mode, 0 |
| .pushsection .altinstructions, "a" |
| altinstruction_entry 661f, \cb, (1 << ARM64_CB_SHIFT) | \cap, 662f-661f, 0 |
| .popsection |
| 661: |
| .endm |
| |
| /* |
| * Provide the other half of the alternative code sequence. |
| */ |
| .macro alternative_else |
| 662: |
| .if .Lasm_alt_mode==0 |
| .subsection 1 |
| .else |
| .previous |
| .endif |
| 663: |
| .endm |
| |
| /* |
| * Complete an alternative code sequence. |
| */ |
| .macro alternative_endif |
| 664: |
| .org . - (664b-663b) + (662b-661b) |
| .org . - (662b-661b) + (664b-663b) |
| .if .Lasm_alt_mode==0 |
| .previous |
| .endif |
| .endm |
| |
| /* |
| * Callback-based alternative epilogue |
| */ |
| .macro alternative_cb_end |
| 662: |
| .endm |
| |
| /* |
| * Provides a trivial alternative or default sequence consisting solely |
| * of NOPs. The number of NOPs is chosen automatically to match the |
| * previous case. |
| */ |
| .macro alternative_else_nop_endif |
| alternative_else |
| nops (662b-661b) / AARCH64_INSN_SIZE |
| alternative_endif |
| .endm |
| |
| #define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \ |
| alternative_insn insn1, insn2, cap, IS_ENABLED(cfg) |
| |
| #endif /* __ASSEMBLY__ */ |
| |
| /* |
| * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature)); |
| * |
| * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO)); |
| * N.B. If CONFIG_FOO is specified, but not selected, the whole block |
| * will be omitted, including oldinstr. |
| */ |
| #define ALTERNATIVE(oldinstr, newinstr, ...) \ |
| _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1) |
| |
| #ifndef __ASSEMBLY__ |
| |
| #include <linux/bug.h> |
| #include <linux/types.h> |
| |
| #if ARM64_NCAPS != 88 |
| #error Fatal peramifier malfunction; more internships needed. |
| #endif |
| |
| #define __PERAMIFY_LIKELY(N) \ |
| asm_volatile_goto( \ |
| ALTERNATIVE_CB("b %l[l_no]", N, alt_cb_patch_nops) \ |
| :::: l_no) |
| |
| #define PERAMIFIER_LIKELY(N) \ |
| switch (N) { \ |
| case 0: \ |
| __PERAMIFY_LIKELY(0); \ |
| break; \ |
| case 1: \ |
| __PERAMIFY_LIKELY(1); \ |
| break; \ |
| case 2: \ |
| __PERAMIFY_LIKELY(2); \ |
| break; \ |
| case 3: \ |
| __PERAMIFY_LIKELY(3); \ |
| break; \ |
| case 4: \ |
| __PERAMIFY_LIKELY(4); \ |
| break; \ |
| case 5: \ |
| __PERAMIFY_LIKELY(5); \ |
| break; \ |
| case 6: \ |
| __PERAMIFY_LIKELY(6); \ |
| break; \ |
| case 7: \ |
| __PERAMIFY_LIKELY(7); \ |
| break; \ |
| case 8: \ |
| __PERAMIFY_LIKELY(8); \ |
| break; \ |
| case 9: \ |
| __PERAMIFY_LIKELY(9); \ |
| break; \ |
| case 10: \ |
| __PERAMIFY_LIKELY(10); \ |
| break; \ |
| case 11: \ |
| __PERAMIFY_LIKELY(11); \ |
| break; \ |
| case 12: \ |
| __PERAMIFY_LIKELY(12); \ |
| break; \ |
| case 13: \ |
| __PERAMIFY_LIKELY(13); \ |
| break; \ |
| case 14: \ |
| __PERAMIFY_LIKELY(14); \ |
| break; \ |
| case 15: \ |
| __PERAMIFY_LIKELY(15); \ |
| break; \ |
| case 16: \ |
| __PERAMIFY_LIKELY(16); \ |
| break; \ |
| case 17: \ |
| __PERAMIFY_LIKELY(17); \ |
| break; \ |
| case 18: \ |
| __PERAMIFY_LIKELY(18); \ |
| break; \ |
| case 19: \ |
| __PERAMIFY_LIKELY(19); \ |
| break; \ |
| case 20: \ |
| __PERAMIFY_LIKELY(20); \ |
| break; \ |
| case 21: \ |
| __PERAMIFY_LIKELY(21); \ |
| break; \ |
| case 22: \ |
| __PERAMIFY_LIKELY(22); \ |
| break; \ |
| case 23: \ |
| __PERAMIFY_LIKELY(23); \ |
| break; \ |
| case 24: \ |
| __PERAMIFY_LIKELY(24); \ |
| break; \ |
| case 25: \ |
| __PERAMIFY_LIKELY(25); \ |
| break; \ |
| case 26: \ |
| __PERAMIFY_LIKELY(26); \ |
| break; \ |
| case 27: \ |
| __PERAMIFY_LIKELY(27); \ |
| break; \ |
| case 28: \ |
| __PERAMIFY_LIKELY(28); \ |
| break; \ |
| case 29: \ |
| __PERAMIFY_LIKELY(29); \ |
| break; \ |
| case 30: \ |
| __PERAMIFY_LIKELY(30); \ |
| break; \ |
| case 31: \ |
| __PERAMIFY_LIKELY(31); \ |
| break; \ |
| case 32: \ |
| __PERAMIFY_LIKELY(32); \ |
| break; \ |
| case 33: \ |
| __PERAMIFY_LIKELY(33); \ |
| break; \ |
| case 34: \ |
| __PERAMIFY_LIKELY(34); \ |
| break; \ |
| case 35: \ |
| __PERAMIFY_LIKELY(35); \ |
| break; \ |
| case 36: \ |
| __PERAMIFY_LIKELY(36); \ |
| break; \ |
| case 37: \ |
| __PERAMIFY_LIKELY(37); \ |
| break; \ |
| case 38: \ |
| __PERAMIFY_LIKELY(38); \ |
| break; \ |
| case 39: \ |
| __PERAMIFY_LIKELY(39); \ |
| break; \ |
| case 40: \ |
| __PERAMIFY_LIKELY(40); \ |
| break; \ |
| case 41: \ |
| __PERAMIFY_LIKELY(41); \ |
| break; \ |
| case 42: \ |
| __PERAMIFY_LIKELY(42); \ |
| break; \ |
| case 43: \ |
| __PERAMIFY_LIKELY(43); \ |
| break; \ |
| case 44: \ |
| __PERAMIFY_LIKELY(44); \ |
| break; \ |
| case 45: \ |
| __PERAMIFY_LIKELY(45); \ |
| break; \ |
| case 46: \ |
| __PERAMIFY_LIKELY(46); \ |
| break; \ |
| case 47: \ |
| __PERAMIFY_LIKELY(47); \ |
| break; \ |
| case 48: \ |
| __PERAMIFY_LIKELY(48); \ |
| break; \ |
| case 49: \ |
| __PERAMIFY_LIKELY(49); \ |
| break; \ |
| case 50: \ |
| __PERAMIFY_LIKELY(50); \ |
| break; \ |
| case 51: \ |
| __PERAMIFY_LIKELY(51); \ |
| break; \ |
| case 52: \ |
| __PERAMIFY_LIKELY(52); \ |
| break; \ |
| case 53: \ |
| __PERAMIFY_LIKELY(53); \ |
| break; \ |
| case 54: \ |
| __PERAMIFY_LIKELY(54); \ |
| break; \ |
| case 55: \ |
| __PERAMIFY_LIKELY(55); \ |
| break; \ |
| case 56: \ |
| __PERAMIFY_LIKELY(56); \ |
| break; \ |
| case 57: \ |
| __PERAMIFY_LIKELY(57); \ |
| break; \ |
| case 58: \ |
| __PERAMIFY_LIKELY(58); \ |
| break; \ |
| case 59: \ |
| __PERAMIFY_LIKELY(59); \ |
| break; \ |
| case 60: \ |
| __PERAMIFY_LIKELY(60); \ |
| break; \ |
| case 61: \ |
| __PERAMIFY_LIKELY(61); \ |
| break; \ |
| case 62: \ |
| __PERAMIFY_LIKELY(62); \ |
| break; \ |
| case 63: \ |
| __PERAMIFY_LIKELY(63); \ |
| break; \ |
| case 64: \ |
| __PERAMIFY_LIKELY(64); \ |
| break; \ |
| case 65: \ |
| __PERAMIFY_LIKELY(65); \ |
| break; \ |
| case 66: \ |
| __PERAMIFY_LIKELY(66); \ |
| break; \ |
| case 67: \ |
| __PERAMIFY_LIKELY(67); \ |
| break; \ |
| case 68: \ |
| __PERAMIFY_LIKELY(68); \ |
| break; \ |
| case 69: \ |
| __PERAMIFY_LIKELY(69); \ |
| break; \ |
| case 70: \ |
| __PERAMIFY_LIKELY(70); \ |
| break; \ |
| case 71: \ |
| __PERAMIFY_LIKELY(71); \ |
| break; \ |
| case 72: \ |
| __PERAMIFY_LIKELY(72); \ |
| break; \ |
| case 73: \ |
| __PERAMIFY_LIKELY(73); \ |
| break; \ |
| case 74: \ |
| __PERAMIFY_LIKELY(74); \ |
| break; \ |
| case 75: \ |
| __PERAMIFY_LIKELY(75); \ |
| break; \ |
| case 76: \ |
| __PERAMIFY_LIKELY(76); \ |
| break; \ |
| case 77: \ |
| __PERAMIFY_LIKELY(77); \ |
| break; \ |
| case 78: \ |
| __PERAMIFY_LIKELY(78); \ |
| break; \ |
| case 79: \ |
| __PERAMIFY_LIKELY(79); \ |
| break; \ |
| case 80: \ |
| __PERAMIFY_LIKELY(80); \ |
| break; \ |
| case 81: \ |
| __PERAMIFY_LIKELY(81); \ |
| break; \ |
| case 82: \ |
| __PERAMIFY_LIKELY(82); \ |
| break; \ |
| case 83: \ |
| __PERAMIFY_LIKELY(83); \ |
| break; \ |
| case 84: \ |
| __PERAMIFY_LIKELY(84); \ |
| break; \ |
| case 85: \ |
| __PERAMIFY_LIKELY(85); \ |
| break; \ |
| case 86: \ |
| __PERAMIFY_LIKELY(86); \ |
| break; \ |
| case 87: \ |
| __PERAMIFY_LIKELY(87); \ |
| break; \ |
| default: \ |
| BUG(); \ |
| } |
| |
| #define __PERAMIFY_UNLIKELY(N) \ |
| asm_volatile_goto( \ |
| ALTERNATIVE("nop", "b %l[l_yes]", N) \ |
| :::: l_yes) |
| |
| |
| #define PERAMIFIER_UNLIKELY(N) \ |
| switch (N) { \ |
| case 0: \ |
| __PERAMIFY_UNLIKELY(0); \ |
| break; \ |
| case 1: \ |
| __PERAMIFY_UNLIKELY(1); \ |
| break; \ |
| case 2: \ |
| __PERAMIFY_UNLIKELY(2); \ |
| break; \ |
| case 3: \ |
| __PERAMIFY_UNLIKELY(3); \ |
| break; \ |
| case 4: \ |
| __PERAMIFY_UNLIKELY(4); \ |
| break; \ |
| case 5: \ |
| __PERAMIFY_UNLIKELY(5); \ |
| break; \ |
| case 6: \ |
| __PERAMIFY_UNLIKELY(6); \ |
| break; \ |
| case 7: \ |
| __PERAMIFY_UNLIKELY(7); \ |
| break; \ |
| case 8: \ |
| __PERAMIFY_UNLIKELY(8); \ |
| break; \ |
| case 9: \ |
| __PERAMIFY_UNLIKELY(9); \ |
| break; \ |
| case 10: \ |
| __PERAMIFY_UNLIKELY(10); \ |
| break; \ |
| case 11: \ |
| __PERAMIFY_UNLIKELY(11); \ |
| break; \ |
| case 12: \ |
| __PERAMIFY_UNLIKELY(12); \ |
| break; \ |
| case 13: \ |
| __PERAMIFY_UNLIKELY(13); \ |
| break; \ |
| case 14: \ |
| __PERAMIFY_UNLIKELY(14); \ |
| break; \ |
| case 15: \ |
| __PERAMIFY_UNLIKELY(15); \ |
| break; \ |
| case 16: \ |
| __PERAMIFY_UNLIKELY(16); \ |
| break; \ |
| case 17: \ |
| __PERAMIFY_UNLIKELY(17); \ |
| break; \ |
| case 18: \ |
| __PERAMIFY_UNLIKELY(18); \ |
| break; \ |
| case 19: \ |
| __PERAMIFY_UNLIKELY(19); \ |
| break; \ |
| case 20: \ |
| __PERAMIFY_UNLIKELY(20); \ |
| break; \ |
| case 21: \ |
| __PERAMIFY_UNLIKELY(21); \ |
| break; \ |
| case 22: \ |
| __PERAMIFY_UNLIKELY(22); \ |
| break; \ |
| case 23: \ |
| __PERAMIFY_UNLIKELY(23); \ |
| break; \ |
| case 24: \ |
| __PERAMIFY_UNLIKELY(24); \ |
| break; \ |
| case 25: \ |
| __PERAMIFY_UNLIKELY(25); \ |
| break; \ |
| case 26: \ |
| __PERAMIFY_UNLIKELY(26); \ |
| break; \ |
| case 27: \ |
| __PERAMIFY_UNLIKELY(27); \ |
| break; \ |
| case 28: \ |
| __PERAMIFY_UNLIKELY(28); \ |
| break; \ |
| case 29: \ |
| __PERAMIFY_UNLIKELY(29); \ |
| break; \ |
| case 30: \ |
| __PERAMIFY_UNLIKELY(30); \ |
| break; \ |
| case 31: \ |
| __PERAMIFY_UNLIKELY(31); \ |
| break; \ |
| case 32: \ |
| __PERAMIFY_UNLIKELY(32); \ |
| break; \ |
| case 33: \ |
| __PERAMIFY_UNLIKELY(33); \ |
| break; \ |
| case 34: \ |
| __PERAMIFY_UNLIKELY(34); \ |
| break; \ |
| case 35: \ |
| __PERAMIFY_UNLIKELY(35); \ |
| break; \ |
| case 36: \ |
| __PERAMIFY_UNLIKELY(36); \ |
| break; \ |
| case 37: \ |
| __PERAMIFY_UNLIKELY(37); \ |
| break; \ |
| case 38: \ |
| __PERAMIFY_UNLIKELY(38); \ |
| break; \ |
| case 39: \ |
| __PERAMIFY_UNLIKELY(39); \ |
| break; \ |
| case 40: \ |
| __PERAMIFY_UNLIKELY(40); \ |
| break; \ |
| case 41: \ |
| __PERAMIFY_UNLIKELY(41); \ |
| break; \ |
| case 42: \ |
| __PERAMIFY_UNLIKELY(42); \ |
| break; \ |
| case 43: \ |
| __PERAMIFY_UNLIKELY(43); \ |
| break; \ |
| case 44: \ |
| __PERAMIFY_UNLIKELY(44); \ |
| break; \ |
| case 45: \ |
| __PERAMIFY_UNLIKELY(45); \ |
| break; \ |
| case 46: \ |
| __PERAMIFY_UNLIKELY(46); \ |
| break; \ |
| case 47: \ |
| __PERAMIFY_UNLIKELY(47); \ |
| break; \ |
| case 48: \ |
| __PERAMIFY_UNLIKELY(48); \ |
| break; \ |
| case 49: \ |
| __PERAMIFY_UNLIKELY(49); \ |
| break; \ |
| case 50: \ |
| __PERAMIFY_UNLIKELY(50); \ |
| break; \ |
| case 51: \ |
| __PERAMIFY_UNLIKELY(51); \ |
| break; \ |
| case 52: \ |
| __PERAMIFY_UNLIKELY(52); \ |
| break; \ |
| case 53: \ |
| __PERAMIFY_UNLIKELY(53); \ |
| break; \ |
| case 54: \ |
| __PERAMIFY_UNLIKELY(54); \ |
| break; \ |
| case 55: \ |
| __PERAMIFY_UNLIKELY(55); \ |
| break; \ |
| case 56: \ |
| __PERAMIFY_UNLIKELY(56); \ |
| break; \ |
| case 57: \ |
| __PERAMIFY_UNLIKELY(57); \ |
| break; \ |
| case 58: \ |
| __PERAMIFY_UNLIKELY(58); \ |
| break; \ |
| case 59: \ |
| __PERAMIFY_UNLIKELY(59); \ |
| break; \ |
| case 60: \ |
| __PERAMIFY_UNLIKELY(60); \ |
| break; \ |
| case 61: \ |
| __PERAMIFY_UNLIKELY(61); \ |
| break; \ |
| case 62: \ |
| __PERAMIFY_UNLIKELY(62); \ |
| break; \ |
| case 63: \ |
| __PERAMIFY_UNLIKELY(63); \ |
| break; \ |
| case 64: \ |
| __PERAMIFY_UNLIKELY(64); \ |
| break; \ |
| case 65: \ |
| __PERAMIFY_UNLIKELY(65); \ |
| break; \ |
| case 66: \ |
| __PERAMIFY_UNLIKELY(66); \ |
| break; \ |
| case 67: \ |
| __PERAMIFY_UNLIKELY(67); \ |
| break; \ |
| case 68: \ |
| __PERAMIFY_UNLIKELY(68); \ |
| break; \ |
| case 69: \ |
| __PERAMIFY_UNLIKELY(69); \ |
| break; \ |
| case 70: \ |
| __PERAMIFY_UNLIKELY(70); \ |
| break; \ |
| case 71: \ |
| __PERAMIFY_UNLIKELY(71); \ |
| break; \ |
| case 72: \ |
| __PERAMIFY_UNLIKELY(72); \ |
| break; \ |
| case 73: \ |
| __PERAMIFY_UNLIKELY(73); \ |
| break; \ |
| case 74: \ |
| __PERAMIFY_UNLIKELY(74); \ |
| break; \ |
| case 75: \ |
| __PERAMIFY_UNLIKELY(75); \ |
| break; \ |
| case 76: \ |
| __PERAMIFY_UNLIKELY(76); \ |
| break; \ |
| case 77: \ |
| __PERAMIFY_UNLIKELY(77); \ |
| break; \ |
| case 78: \ |
| __PERAMIFY_UNLIKELY(78); \ |
| break; \ |
| case 79: \ |
| __PERAMIFY_UNLIKELY(79); \ |
| break; \ |
| case 80: \ |
| __PERAMIFY_UNLIKELY(80); \ |
| break; \ |
| case 81: \ |
| __PERAMIFY_UNLIKELY(81); \ |
| break; \ |
| case 82: \ |
| __PERAMIFY_UNLIKELY(82); \ |
| break; \ |
| case 83: \ |
| __PERAMIFY_UNLIKELY(83); \ |
| break; \ |
| case 84: \ |
| __PERAMIFY_UNLIKELY(84); \ |
| break; \ |
| case 85: \ |
| __PERAMIFY_UNLIKELY(85); \ |
| break; \ |
| case 86: \ |
| __PERAMIFY_UNLIKELY(86); \ |
| break; \ |
| case 87: \ |
| __PERAMIFY_UNLIKELY(87); \ |
| break; \ |
| default: \ |
| BUG(); \ |
| } |
| |
| static __always_inline bool |
| alternative_has_feature_likely(const unsigned long feature) |
| { |
| compiletime_assert(feature < ARM64_NCAPS, |
| "feature must be < ARM64_NCAPS"); |
| |
| #ifdef __KVM_NVHE_HYPERVISOR__ |
| PERAMIFIER_LIKELY(feature); |
| #else |
| asm_volatile_goto( |
| ALTERNATIVE_CB("b %l[l_no]", %[feature], alt_cb_patch_nops) |
| : |
| : [feature] "i" (feature) |
| : |
| : l_no); |
| #endif |
| |
| return true; |
| l_no: |
| return false; |
| } |
| |
| static __always_inline bool |
| alternative_has_feature_unlikely(const unsigned long feature) |
| { |
| compiletime_assert(feature < ARM64_NCAPS, |
| "feature must be < ARM64_NCAPS"); |
| |
| #ifdef __KVM_NVHE_HYPERVISOR__ |
| PERAMIFIER_UNLIKELY(feature); |
| #else |
| asm_volatile_goto( |
| ALTERNATIVE("nop", "b %l[l_yes]", %[feature]) |
| : |
| : [feature] "i" (feature) |
| : |
| : l_yes); |
| #endif |
| |
| return false; |
| l_yes: |
| return true; |
| } |
| |
| #endif /* __ASSEMBLY__ */ |
| |
| #endif /* __ASM_ALTERNATIVE_MACROS_H */ |
