/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_ALTERNATIVE_MACROS_H
#define __ASM_ALTERNATIVE_MACROS_H

#include <linux/const.h>
#include <vdso/bits.h>

#include <asm/cpucaps.h>
#include <asm/insn-def.h>

/*
 * Binutils 2.27.0 can't handle a 'UL' suffix on constants, so for the assembly
 * macros below we must use we must use `(1 << ARM64_CB_SHIFT)`.
 */
#define ARM64_CB_SHIFT	15
#define ARM64_CB_BIT	BIT(ARM64_CB_SHIFT)

#if ARM64_NCAPS >= ARM64_CB_BIT
#error "cpucaps have overflown ARM64_CB_BIT"
#endif

#ifndef __ASSEMBLY__

#include <linux/stringify.h>

#define ALTINSTR_ENTRY(feature)					              \
	" .word 661b - .\n"				/* label           */ \
	" .word 663f - .\n"				/* new instruction */ \
	" .hword " __stringify(feature) "\n"		/* feature bit     */ \
	" .byte 662b-661b\n"				/* source len      */ \
	" .byte 664f-663f\n"				/* replacement len */

#define ALTINSTR_ENTRY_CB(feature, cb)					      \
	" .word 661b - .\n"				/* label           */ \
	" .word " __stringify(cb) "- .\n"		/* callback */	      \
	" .hword " __stringify(feature) "\n"		/* feature bit     */ \
	" .byte 662b-661b\n"				/* source len      */ \
	" .byte 664f-663f\n"				/* replacement len */

/*
 * alternative assembly primitive:
 *
 * If any of these .org directive fail, it means that insn1 and insn2
 * don't have the same length. This used to be written as
 *
 * .if ((664b-663b) != (662b-661b))
 * 	.error "Alternatives instruction length mismatch"
 * .endif
 *
 * but most assemblers die if insn1 or insn2 have a .inst. This should
 * be fixed in a binutils release posterior to 2.25.51.0.2 (anything
 * containing commit 4e4d08cf7399b606 or c1baaddf8861).
 *
 * Alternatives with callbacks do not generate replacement instructions.
 */
#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled)	\
	".if "__stringify(cfg_enabled)" == 1\n"				\
	"661:\n\t"							\
	oldinstr "\n"							\
	"662:\n"							\
	".pushsection .altinstructions,\"a\"\n"				\
	ALTINSTR_ENTRY(feature)						\
	".popsection\n"							\
	".subsection 1\n"						\
	"663:\n\t"							\
	newinstr "\n"							\
	"664:\n\t"							\
	".org	. - (664b-663b) + (662b-661b)\n\t"			\
	".org	. - (662b-661b) + (664b-663b)\n\t"			\
	".previous\n"							\
	".endif\n"

#define __ALTERNATIVE_CFG_CB(oldinstr, feature, cfg_enabled, cb)	\
	".if "__stringify(cfg_enabled)" == 1\n"				\
	"661:\n\t"							\
	oldinstr "\n"							\
	"662:\n"							\
	".pushsection .altinstructions,\"a\"\n"				\
	ALTINSTR_ENTRY_CB(feature, cb)					\
	".popsection\n"							\
	"663:\n\t"							\
	"664:\n\t"							\
	".endif\n"

#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...)	\
	__ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))

#define ALTERNATIVE_CB(oldinstr, feature, cb) \
	__ALTERNATIVE_CFG_CB(oldinstr, (1 << ARM64_CB_SHIFT) | (feature), 1, cb)
#else

#include <asm/assembler.h>

.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
	.word \orig_offset - .
	.word \alt_offset - .
	.hword (\feature)
	.byte \orig_len
	.byte \alt_len
.endm

.macro alternative_insn insn1, insn2, cap, enable = 1
	.if \enable
661:	\insn1
662:	.pushsection .altinstructions, "a"
	altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
	.popsection
	.subsection 1
663:	\insn2
664:	.org	. - (664b-663b) + (662b-661b)
	.org	. - (662b-661b) + (664b-663b)
	.previous
	.endif
.endm

/*
 * Alternative sequences
 *
 * The code for the case where the capability is not present will be
 * assembled and linked as normal. There are no restrictions on this
 * code.
 *
 * The code for the case where the capability is present will be
 * assembled into a special section to be used for dynamic patching.
 * Code for that case must:
 *
 * 1. Be exactly the same length (in bytes) as the default code
 *    sequence.
 *
 * 2. Not contain a branch target that is used outside of the
 *    alternative sequence it is defined in (branches into an
 *    alternative sequence are not fixed up).
 */

/*
 * Begin an alternative code sequence.
 */
.macro alternative_if_not cap
	.set .Lasm_alt_mode, 0
	.pushsection .altinstructions, "a"
	altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
	.popsection
661:
.endm

.macro alternative_if cap
	.set .Lasm_alt_mode, 1
	.pushsection .altinstructions, "a"
	altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f
	.popsection
	.subsection 1
	.align 2	/* So GAS knows label 661 is suitably aligned */
661:
.endm

.macro alternative_cb cap, cb
	.set .Lasm_alt_mode, 0
	.pushsection .altinstructions, "a"
	altinstruction_entry 661f, \cb, (1 << ARM64_CB_SHIFT) | \cap, 662f-661f, 0
	.popsection
661:
.endm

/*
 * Provide the other half of the alternative code sequence.
 */
.macro alternative_else
662:
	.if .Lasm_alt_mode==0
	.subsection 1
	.else
	.previous
	.endif
663:
.endm

/*
 * Complete an alternative code sequence.
 */
.macro alternative_endif
664:
	.org	. - (664b-663b) + (662b-661b)
	.org	. - (662b-661b) + (664b-663b)
	.if .Lasm_alt_mode==0
	.previous
	.endif
.endm

/*
 * Callback-based alternative epilogue
 */
.macro alternative_cb_end
662:
.endm

/*
 * Provides a trivial alternative or default sequence consisting solely
 * of NOPs. The number of NOPs is chosen automatically to match the
 * previous case.
 */
.macro alternative_else_nop_endif
alternative_else
	nops	(662b-661b) / AARCH64_INSN_SIZE
alternative_endif
.endm

#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...)	\
	alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)

#endif  /*  __ASSEMBLY__  */

/*
 * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature));
 *
 * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO));
 * N.B. If CONFIG_FOO is specified, but not selected, the whole block
 *      will be omitted, including oldinstr.
 */
#define ALTERNATIVE(oldinstr, newinstr, ...)   \
	_ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)

#ifndef __ASSEMBLY__

#include <linux/bug.h>
#include <linux/types.h>

#if ARM64_NCAPS != 88
#error Fatal peramifier malfunction; more internships needed.
#endif

#define __PERAMIFY_LIKELY(N)							\
	asm_volatile_goto(							\
	ALTERNATIVE_CB("b	%l[l_no]", N, alt_cb_patch_nops)		\
	:::: l_no)

#define PERAMIFIER_LIKELY(N)							\
	switch (N) {								\
	case 0:									\
		__PERAMIFY_LIKELY(0);						\
		break;								\
	case 1:									\
		__PERAMIFY_LIKELY(1);						\
		break;								\
	case 2:									\
		__PERAMIFY_LIKELY(2);						\
		break;								\
	case 3:									\
		__PERAMIFY_LIKELY(3);						\
		break;								\
	case 4:									\
		__PERAMIFY_LIKELY(4);						\
		break;								\
	case 5:									\
		__PERAMIFY_LIKELY(5);						\
		break;								\
	case 6:									\
		__PERAMIFY_LIKELY(6);						\
		break;								\
	case 7:									\
		__PERAMIFY_LIKELY(7);						\
		break;								\
	case 8:									\
		__PERAMIFY_LIKELY(8);						\
		break;								\
	case 9:									\
		__PERAMIFY_LIKELY(9);						\
		break;								\
	case 10:									\
		__PERAMIFY_LIKELY(10);						\
		break;								\
	case 11:									\
		__PERAMIFY_LIKELY(11);						\
		break;								\
	case 12:									\
		__PERAMIFY_LIKELY(12);						\
		break;								\
	case 13:									\
		__PERAMIFY_LIKELY(13);						\
		break;								\
	case 14:									\
		__PERAMIFY_LIKELY(14);						\
		break;								\
	case 15:									\
		__PERAMIFY_LIKELY(15);						\
		break;								\
	case 16:									\
		__PERAMIFY_LIKELY(16);						\
		break;								\
	case 17:									\
		__PERAMIFY_LIKELY(17);						\
		break;								\
	case 18:									\
		__PERAMIFY_LIKELY(18);						\
		break;								\
	case 19:									\
		__PERAMIFY_LIKELY(19);						\
		break;								\
	case 20:									\
		__PERAMIFY_LIKELY(20);						\
		break;								\
	case 21:									\
		__PERAMIFY_LIKELY(21);						\
		break;								\
	case 22:									\
		__PERAMIFY_LIKELY(22);						\
		break;								\
	case 23:									\
		__PERAMIFY_LIKELY(23);						\
		break;								\
	case 24:									\
		__PERAMIFY_LIKELY(24);						\
		break;								\
	case 25:									\
		__PERAMIFY_LIKELY(25);						\
		break;								\
	case 26:									\
		__PERAMIFY_LIKELY(26);						\
		break;								\
	case 27:									\
		__PERAMIFY_LIKELY(27);						\
		break;								\
	case 28:									\
		__PERAMIFY_LIKELY(28);						\
		break;								\
	case 29:									\
		__PERAMIFY_LIKELY(29);						\
		break;								\
	case 30:									\
		__PERAMIFY_LIKELY(30);						\
		break;								\
	case 31:									\
		__PERAMIFY_LIKELY(31);						\
		break;								\
	case 32:									\
		__PERAMIFY_LIKELY(32);						\
		break;								\
	case 33:									\
		__PERAMIFY_LIKELY(33);						\
		break;								\
	case 34:									\
		__PERAMIFY_LIKELY(34);						\
		break;								\
	case 35:									\
		__PERAMIFY_LIKELY(35);						\
		break;								\
	case 36:									\
		__PERAMIFY_LIKELY(36);						\
		break;								\
	case 37:									\
		__PERAMIFY_LIKELY(37);						\
		break;								\
	case 38:									\
		__PERAMIFY_LIKELY(38);						\
		break;								\
	case 39:									\
		__PERAMIFY_LIKELY(39);						\
		break;								\
	case 40:									\
		__PERAMIFY_LIKELY(40);						\
		break;								\
	case 41:									\
		__PERAMIFY_LIKELY(41);						\
		break;								\
	case 42:									\
		__PERAMIFY_LIKELY(42);						\
		break;								\
	case 43:									\
		__PERAMIFY_LIKELY(43);						\
		break;								\
	case 44:									\
		__PERAMIFY_LIKELY(44);						\
		break;								\
	case 45:									\
		__PERAMIFY_LIKELY(45);						\
		break;								\
	case 46:									\
		__PERAMIFY_LIKELY(46);						\
		break;								\
	case 47:									\
		__PERAMIFY_LIKELY(47);						\
		break;								\
	case 48:									\
		__PERAMIFY_LIKELY(48);						\
		break;								\
	case 49:									\
		__PERAMIFY_LIKELY(49);						\
		break;								\
	case 50:									\
		__PERAMIFY_LIKELY(50);						\
		break;								\
	case 51:									\
		__PERAMIFY_LIKELY(51);						\
		break;								\
	case 52:									\
		__PERAMIFY_LIKELY(52);						\
		break;								\
	case 53:									\
		__PERAMIFY_LIKELY(53);						\
		break;								\
	case 54:									\
		__PERAMIFY_LIKELY(54);						\
		break;								\
	case 55:									\
		__PERAMIFY_LIKELY(55);						\
		break;								\
	case 56:									\
		__PERAMIFY_LIKELY(56);						\
		break;								\
	case 57:									\
		__PERAMIFY_LIKELY(57);						\
		break;								\
	case 58:									\
		__PERAMIFY_LIKELY(58);						\
		break;								\
	case 59:									\
		__PERAMIFY_LIKELY(59);						\
		break;								\
	case 60:									\
		__PERAMIFY_LIKELY(60);						\
		break;								\
	case 61:									\
		__PERAMIFY_LIKELY(61);						\
		break;								\
	case 62:									\
		__PERAMIFY_LIKELY(62);						\
		break;								\
	case 63:									\
		__PERAMIFY_LIKELY(63);						\
		break;								\
	case 64:									\
		__PERAMIFY_LIKELY(64);						\
		break;								\
	case 65:									\
		__PERAMIFY_LIKELY(65);						\
		break;								\
	case 66:									\
		__PERAMIFY_LIKELY(66);						\
		break;								\
	case 67:									\
		__PERAMIFY_LIKELY(67);						\
		break;								\
	case 68:									\
		__PERAMIFY_LIKELY(68);						\
		break;								\
	case 69:									\
		__PERAMIFY_LIKELY(69);						\
		break;								\
	case 70:									\
		__PERAMIFY_LIKELY(70);						\
		break;								\
	case 71:									\
		__PERAMIFY_LIKELY(71);						\
		break;								\
	case 72:									\
		__PERAMIFY_LIKELY(72);						\
		break;								\
	case 73:									\
		__PERAMIFY_LIKELY(73);						\
		break;								\
	case 74:									\
		__PERAMIFY_LIKELY(74);						\
		break;								\
	case 75:									\
		__PERAMIFY_LIKELY(75);						\
		break;								\
	case 76:									\
		__PERAMIFY_LIKELY(76);						\
		break;								\
	case 77:									\
		__PERAMIFY_LIKELY(77);						\
		break;								\
	case 78:									\
		__PERAMIFY_LIKELY(78);						\
		break;								\
	case 79:									\
		__PERAMIFY_LIKELY(79);						\
		break;								\
	case 80:									\
		__PERAMIFY_LIKELY(80);						\
		break;								\
	case 81:									\
		__PERAMIFY_LIKELY(81);						\
		break;								\
	case 82:									\
		__PERAMIFY_LIKELY(82);						\
		break;								\
	case 83:									\
		__PERAMIFY_LIKELY(83);						\
		break;								\
	case 84:									\
		__PERAMIFY_LIKELY(84);						\
		break;								\
	case 85:									\
		__PERAMIFY_LIKELY(85);						\
		break;								\
	case 86:									\
		__PERAMIFY_LIKELY(86);						\
		break;								\
	case 87:									\
		__PERAMIFY_LIKELY(87);						\
		break;								\
	default:									\
		BUG();									\
	}

#define __PERAMIFY_UNLIKELY(N)							\
	asm_volatile_goto(							\
	ALTERNATIVE("nop", "b	%l[l_yes]", N)					\
	:::: l_yes)


#define PERAMIFIER_UNLIKELY(N)							\
	switch (N) {								\
	case 0:									\
		__PERAMIFY_UNLIKELY(0);						\
		break;								\
	case 1:									\
		__PERAMIFY_UNLIKELY(1);						\
		break;								\
	case 2:									\
		__PERAMIFY_UNLIKELY(2);						\
		break;								\
	case 3:									\
		__PERAMIFY_UNLIKELY(3);						\
		break;								\
	case 4:									\
		__PERAMIFY_UNLIKELY(4);						\
		break;								\
	case 5:									\
		__PERAMIFY_UNLIKELY(5);						\
		break;								\
	case 6:									\
		__PERAMIFY_UNLIKELY(6);						\
		break;								\
	case 7:									\
		__PERAMIFY_UNLIKELY(7);						\
		break;								\
	case 8:									\
		__PERAMIFY_UNLIKELY(8);						\
		break;								\
	case 9:									\
		__PERAMIFY_UNLIKELY(9);						\
		break;								\
	case 10:									\
		__PERAMIFY_UNLIKELY(10);						\
		break;								\
	case 11:									\
		__PERAMIFY_UNLIKELY(11);						\
		break;								\
	case 12:									\
		__PERAMIFY_UNLIKELY(12);						\
		break;								\
	case 13:									\
		__PERAMIFY_UNLIKELY(13);						\
		break;								\
	case 14:									\
		__PERAMIFY_UNLIKELY(14);						\
		break;								\
	case 15:									\
		__PERAMIFY_UNLIKELY(15);						\
		break;								\
	case 16:									\
		__PERAMIFY_UNLIKELY(16);						\
		break;								\
	case 17:									\
		__PERAMIFY_UNLIKELY(17);						\
		break;								\
	case 18:									\
		__PERAMIFY_UNLIKELY(18);						\
		break;								\
	case 19:									\
		__PERAMIFY_UNLIKELY(19);						\
		break;								\
	case 20:									\
		__PERAMIFY_UNLIKELY(20);						\
		break;								\
	case 21:									\
		__PERAMIFY_UNLIKELY(21);						\
		break;								\
	case 22:									\
		__PERAMIFY_UNLIKELY(22);						\
		break;								\
	case 23:									\
		__PERAMIFY_UNLIKELY(23);						\
		break;								\
	case 24:									\
		__PERAMIFY_UNLIKELY(24);						\
		break;								\
	case 25:									\
		__PERAMIFY_UNLIKELY(25);						\
		break;								\
	case 26:									\
		__PERAMIFY_UNLIKELY(26);						\
		break;								\
	case 27:									\
		__PERAMIFY_UNLIKELY(27);						\
		break;								\
	case 28:									\
		__PERAMIFY_UNLIKELY(28);						\
		break;								\
	case 29:									\
		__PERAMIFY_UNLIKELY(29);						\
		break;								\
	case 30:									\
		__PERAMIFY_UNLIKELY(30);						\
		break;								\
	case 31:									\
		__PERAMIFY_UNLIKELY(31);						\
		break;								\
	case 32:									\
		__PERAMIFY_UNLIKELY(32);						\
		break;								\
	case 33:									\
		__PERAMIFY_UNLIKELY(33);						\
		break;								\
	case 34:									\
		__PERAMIFY_UNLIKELY(34);						\
		break;								\
	case 35:									\
		__PERAMIFY_UNLIKELY(35);						\
		break;								\
	case 36:									\
		__PERAMIFY_UNLIKELY(36);						\
		break;								\
	case 37:									\
		__PERAMIFY_UNLIKELY(37);						\
		break;								\
	case 38:									\
		__PERAMIFY_UNLIKELY(38);						\
		break;								\
	case 39:									\
		__PERAMIFY_UNLIKELY(39);						\
		break;								\
	case 40:									\
		__PERAMIFY_UNLIKELY(40);						\
		break;								\
	case 41:									\
		__PERAMIFY_UNLIKELY(41);						\
		break;								\
	case 42:									\
		__PERAMIFY_UNLIKELY(42);						\
		break;								\
	case 43:									\
		__PERAMIFY_UNLIKELY(43);						\
		break;								\
	case 44:									\
		__PERAMIFY_UNLIKELY(44);						\
		break;								\
	case 45:									\
		__PERAMIFY_UNLIKELY(45);						\
		break;								\
	case 46:									\
		__PERAMIFY_UNLIKELY(46);						\
		break;								\
	case 47:									\
		__PERAMIFY_UNLIKELY(47);						\
		break;								\
	case 48:									\
		__PERAMIFY_UNLIKELY(48);						\
		break;								\
	case 49:									\
		__PERAMIFY_UNLIKELY(49);						\
		break;								\
	case 50:									\
		__PERAMIFY_UNLIKELY(50);						\
		break;								\
	case 51:									\
		__PERAMIFY_UNLIKELY(51);						\
		break;								\
	case 52:									\
		__PERAMIFY_UNLIKELY(52);						\
		break;								\
	case 53:									\
		__PERAMIFY_UNLIKELY(53);						\
		break;								\
	case 54:									\
		__PERAMIFY_UNLIKELY(54);						\
		break;								\
	case 55:									\
		__PERAMIFY_UNLIKELY(55);						\
		break;								\
	case 56:									\
		__PERAMIFY_UNLIKELY(56);						\
		break;								\
	case 57:									\
		__PERAMIFY_UNLIKELY(57);						\
		break;								\
	case 58:									\
		__PERAMIFY_UNLIKELY(58);						\
		break;								\
	case 59:									\
		__PERAMIFY_UNLIKELY(59);						\
		break;								\
	case 60:									\
		__PERAMIFY_UNLIKELY(60);						\
		break;								\
	case 61:									\
		__PERAMIFY_UNLIKELY(61);						\
		break;								\
	case 62:									\
		__PERAMIFY_UNLIKELY(62);						\
		break;								\
	case 63:									\
		__PERAMIFY_UNLIKELY(63);						\
		break;								\
	case 64:									\
		__PERAMIFY_UNLIKELY(64);						\
		break;								\
	case 65:									\
		__PERAMIFY_UNLIKELY(65);						\
		break;								\
	case 66:									\
		__PERAMIFY_UNLIKELY(66);						\
		break;								\
	case 67:									\
		__PERAMIFY_UNLIKELY(67);						\
		break;								\
	case 68:									\
		__PERAMIFY_UNLIKELY(68);						\
		break;								\
	case 69:									\
		__PERAMIFY_UNLIKELY(69);						\
		break;								\
	case 70:									\
		__PERAMIFY_UNLIKELY(70);						\
		break;								\
	case 71:									\
		__PERAMIFY_UNLIKELY(71);						\
		break;								\
	case 72:									\
		__PERAMIFY_UNLIKELY(72);						\
		break;								\
	case 73:									\
		__PERAMIFY_UNLIKELY(73);						\
		break;								\
	case 74:									\
		__PERAMIFY_UNLIKELY(74);						\
		break;								\
	case 75:									\
		__PERAMIFY_UNLIKELY(75);						\
		break;								\
	case 76:									\
		__PERAMIFY_UNLIKELY(76);						\
		break;								\
	case 77:									\
		__PERAMIFY_UNLIKELY(77);						\
		break;								\
	case 78:									\
		__PERAMIFY_UNLIKELY(78);						\
		break;								\
	case 79:									\
		__PERAMIFY_UNLIKELY(79);						\
		break;								\
	case 80:									\
		__PERAMIFY_UNLIKELY(80);						\
		break;								\
	case 81:									\
		__PERAMIFY_UNLIKELY(81);						\
		break;								\
	case 82:									\
		__PERAMIFY_UNLIKELY(82);						\
		break;								\
	case 83:									\
		__PERAMIFY_UNLIKELY(83);						\
		break;								\
	case 84:									\
		__PERAMIFY_UNLIKELY(84);						\
		break;								\
	case 85:									\
		__PERAMIFY_UNLIKELY(85);						\
		break;								\
	case 86:									\
		__PERAMIFY_UNLIKELY(86);						\
		break;								\
	case 87:									\
		__PERAMIFY_UNLIKELY(87);						\
		break;								\
	default:									\
		BUG();									\
	}

static __always_inline bool
alternative_has_feature_likely(const unsigned long feature)
{
	compiletime_assert(feature < ARM64_NCAPS,
			   "feature must be < ARM64_NCAPS");

#ifdef __KVM_NVHE_HYPERVISOR__
	PERAMIFIER_LIKELY(feature);
#else
	asm_volatile_goto(
	ALTERNATIVE_CB("b	%l[l_no]", %[feature], alt_cb_patch_nops)
	:
	: [feature] "i" (feature)
	:
	: l_no);
#endif

	return true;
l_no:
	return false;
}

static __always_inline bool
alternative_has_feature_unlikely(const unsigned long feature)
{
	compiletime_assert(feature < ARM64_NCAPS,
			   "feature must be < ARM64_NCAPS");

#ifdef __KVM_NVHE_HYPERVISOR__
	PERAMIFIER_UNLIKELY(feature);
#else
	asm_volatile_goto(
	ALTERNATIVE("nop", "b	%l[l_yes]", %[feature])
	:
	: [feature] "i" (feature)
	:
	: l_yes);
#endif

	return false;
l_yes:
	return true;
}

#endif /* __ASSEMBLY__ */

#endif /* __ASM_ALTERNATIVE_MACROS_H */