/* SPDX-License-Identifier: GPL-2.0 */

#include <linux/stringify.h>
#include <linux/linkage.h>
#include <asm/alternative.h>
#include <asm/dwarf.h>
#include <asm/fpu-insn.h>

#define STATE0	%v0
#define STATE1	%v1
#define STATE2	%v2
#define STATE3	%v3
#define COPY0	%v4
#define COPY1	%v5
#define COPY2	%v6
#define COPY3	%v7
#define BEPERM	%v19
#define TMP0	%v20
#define TMP1	%v21
#define TMP2	%v22
#define TMP3	%v23

	.section .rodata

	.balign 32
SYM_DATA_START_LOCAL(chacha20_constants)
	.long	0x61707865,0x3320646e,0x79622d32,0x6b206574 # endian-neutral
	.long	0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c # byte swap
SYM_DATA_END(chacha20_constants)

	.text
/*
 * s390 ChaCha20 implementation meant for vDSO. Produces a given positive
 * number of blocks of output with nonce 0, taking an input key and 8-bytes
 * counter. Does not spill to the stack.
 *
 * void __arch_chacha20_blocks_nostack(uint8_t *dst_bytes,
 *				       const uint8_t *key,
 *				       uint32_t *counter,
 *				       size_t nblocks)
 */
SYM_FUNC_START(__arch_chacha20_blocks_nostack)
	CFI_STARTPROC
	larl	%r1,chacha20_constants

	/* COPY0 = "expand 32-byte k" */
	VL	COPY0,0,,%r1

	/* BEPERM = byte selectors for VPERM */
	ALTERNATIVE __stringify(VL BEPERM,16,,%r1), "brcl 0,0", ALT_FACILITY(148)

	/* COPY1,COPY2 = key */
	VLM	COPY1,COPY2,0,%r3

	/* COPY3 = counter || zero nonce  */
	lg	%r3,0(%r4)
	VZERO	COPY3
	VLVGG	COPY3,%r3,0

	lghi	%r1,0
.Lblock:
	VLR	STATE0,COPY0
	VLR	STATE1,COPY1
	VLR	STATE2,COPY2
	VLR	STATE3,COPY3

	lghi	%r0,10
.Ldoubleround:
	/* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */
	VAF	STATE0,STATE0,STATE1
	VX	STATE3,STATE3,STATE0
	VERLLF	STATE3,STATE3,16

	/* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */
	VAF	STATE2,STATE2,STATE3
	VX	STATE1,STATE1,STATE2
	VERLLF	STATE1,STATE1,12

	/* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */
	VAF	STATE0,STATE0,STATE1
	VX	STATE3,STATE3,STATE0
	VERLLF	STATE3,STATE3,8

	/* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */
	VAF	STATE2,STATE2,STATE3
	VX	STATE1,STATE1,STATE2
	VERLLF	STATE1,STATE1,7

	/* STATE1[0,1,2,3] = STATE1[1,2,3,0] */
	VSLDB	STATE1,STATE1,STATE1,4
	/* STATE2[0,1,2,3] = STATE2[2,3,0,1] */
	VSLDB	STATE2,STATE2,STATE2,8
	/* STATE3[0,1,2,3] = STATE3[3,0,1,2] */
	VSLDB	STATE3,STATE3,STATE3,12

	/* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */
	VAF	STATE0,STATE0,STATE1
	VX	STATE3,STATE3,STATE0
	VERLLF	STATE3,STATE3,16

	/* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */
	VAF	STATE2,STATE2,STATE3
	VX	STATE1,STATE1,STATE2
	VERLLF	STATE1,STATE1,12

	/* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */
	VAF	STATE0,STATE0,STATE1
	VX	STATE3,STATE3,STATE0
	VERLLF	STATE3,STATE3,8

	/* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */
	VAF	STATE2,STATE2,STATE3
	VX	STATE1,STATE1,STATE2
	VERLLF	STATE1,STATE1,7

	/* STATE1[0,1,2,3] = STATE1[3,0,1,2] */
	VSLDB	STATE1,STATE1,STATE1,12
	/* STATE2[0,1,2,3] = STATE2[2,3,0,1] */
	VSLDB	STATE2,STATE2,STATE2,8
	/* STATE3[0,1,2,3] = STATE3[1,2,3,0] */
	VSLDB	STATE3,STATE3,STATE3,4
	brctg	%r0,.Ldoubleround

	/* OUTPUT0 = STATE0 + COPY0 */
	VAF	STATE0,STATE0,COPY0
	/* OUTPUT1 = STATE1 + COPY1 */
	VAF	STATE1,STATE1,COPY1
	/* OUTPUT2 = STATE2 + COPY2 */
	VAF	STATE2,STATE2,COPY2
	/* OUTPUT3 = STATE3 + COPY3 */
	VAF	STATE3,STATE3,COPY3

	ALTERNATIVE							\
		__stringify(						\
		/* Convert STATE to little endian and store to OUTPUT */\
		VPERM	TMP0,STATE0,STATE0,BEPERM;			\
		VPERM	TMP1,STATE1,STATE1,BEPERM;			\
		VPERM	TMP2,STATE2,STATE2,BEPERM;			\
		VPERM	TMP3,STATE3,STATE3,BEPERM;			\
		VSTM	TMP0,TMP3,0,%r2),				\
		__stringify(						\
		/* 32 bit wise little endian store to OUTPUT */		\
		VSTBRF	STATE0,0,,%r2;					\
		VSTBRF	STATE1,16,,%r2;					\
		VSTBRF	STATE2,32,,%r2;					\
		VSTBRF	STATE3,48,,%r2;					\
		brcl	0,0),						\
		ALT_FACILITY(148)

	/* ++COPY3.COUNTER */
	/* alsih %r3,1 */
	.insn	rilu,0xcc0a00000000,%r3,1
	alcr	%r3,%r1
	VLVGG	COPY3,%r3,0

	/* OUTPUT += 64, --NBLOCKS */
	aghi	%r2,64
	brctg	%r5,.Lblock

	/* COUNTER = COPY3.COUNTER */
	stg	%r3,0(%r4)

	/* Zero out potentially sensitive regs */
	VZERO	STATE0
	VZERO	STATE1
	VZERO	STATE2
	VZERO	STATE3
	VZERO	COPY1
	VZERO	COPY2

	/* Early exit if TMP0-TMP3 have not been used */
	ALTERNATIVE "nopr", "br %r14", ALT_FACILITY(148)

	VZERO	TMP0
	VZERO	TMP1
	VZERO	TMP2
	VZERO	TMP3

	br	%r14
	CFI_ENDPROC
SYM_FUNC_END(__arch_chacha20_blocks_nostack)