| /* SPDX-License-Identifier: GPL-2.0 */ | |
| #include <linux/stringify.h> | |
| #include <linux/linkage.h> | |
| #include <asm/alternative.h> | |
| #include <asm/dwarf.h> | |
| #include <asm/fpu-insn.h> | |
| #define STATE0 %v0 | |
| #define STATE1 %v1 | |
| #define STATE2 %v2 | |
| #define STATE3 %v3 | |
| #define COPY0 %v4 | |
| #define COPY1 %v5 | |
| #define COPY2 %v6 | |
| #define COPY3 %v7 | |
| #define BEPERM %v19 | |
| #define TMP0 %v20 | |
| #define TMP1 %v21 | |
| #define TMP2 %v22 | |
| #define TMP3 %v23 | |
| .section .rodata | |
| .balign 32 | |
| SYM_DATA_START_LOCAL(chacha20_constants) | |
| .long 0x61707865,0x3320646e,0x79622d32,0x6b206574 # endian-neutral | |
| .long 0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c # byte swap | |
| SYM_DATA_END(chacha20_constants) | |
| .text | |
| /* | |
| * s390 ChaCha20 implementation meant for vDSO. Produces a given positive | |
| * number of blocks of output with nonce 0, taking an input key and 8-bytes | |
| * counter. Does not spill to the stack. | |
| * | |
| * void __arch_chacha20_blocks_nostack(uint8_t *dst_bytes, | |
| * const uint8_t *key, | |
| * uint32_t *counter, | |
| * size_t nblocks) | |
| */ | |
| SYM_FUNC_START(__arch_chacha20_blocks_nostack) | |
| CFI_STARTPROC | |
| larl %r1,chacha20_constants | |
| /* COPY0 = "expand 32-byte k" */ | |
| VL COPY0,0,,%r1 | |
| /* BEPERM = byte selectors for VPERM */ | |
| ALTERNATIVE __stringify(VL BEPERM,16,,%r1), "brcl 0,0", ALT_FACILITY(148) | |
| /* COPY1,COPY2 = key */ | |
| VLM COPY1,COPY2,0,%r3 | |
| /* COPY3 = counter || zero nonce */ | |
| lg %r3,0(%r4) | |
| VZERO COPY3 | |
| VLVGG COPY3,%r3,0 | |
| lghi %r1,0 | |
| .Lblock: | |
| VLR STATE0,COPY0 | |
| VLR STATE1,COPY1 | |
| VLR STATE2,COPY2 | |
| VLR STATE3,COPY3 | |
| lghi %r0,10 | |
| .Ldoubleround: | |
| /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */ | |
| VAF STATE0,STATE0,STATE1 | |
| VX STATE3,STATE3,STATE0 | |
| VERLLF STATE3,STATE3,16 | |
| /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */ | |
| VAF STATE2,STATE2,STATE3 | |
| VX STATE1,STATE1,STATE2 | |
| VERLLF STATE1,STATE1,12 | |
| /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */ | |
| VAF STATE0,STATE0,STATE1 | |
| VX STATE3,STATE3,STATE0 | |
| VERLLF STATE3,STATE3,8 | |
| /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */ | |
| VAF STATE2,STATE2,STATE3 | |
| VX STATE1,STATE1,STATE2 | |
| VERLLF STATE1,STATE1,7 | |
| /* STATE1[0,1,2,3] = STATE1[1,2,3,0] */ | |
| VSLDB STATE1,STATE1,STATE1,4 | |
| /* STATE2[0,1,2,3] = STATE2[2,3,0,1] */ | |
| VSLDB STATE2,STATE2,STATE2,8 | |
| /* STATE3[0,1,2,3] = STATE3[3,0,1,2] */ | |
| VSLDB STATE3,STATE3,STATE3,12 | |
| /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */ | |
| VAF STATE0,STATE0,STATE1 | |
| VX STATE3,STATE3,STATE0 | |
| VERLLF STATE3,STATE3,16 | |
| /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */ | |
| VAF STATE2,STATE2,STATE3 | |
| VX STATE1,STATE1,STATE2 | |
| VERLLF STATE1,STATE1,12 | |
| /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */ | |
| VAF STATE0,STATE0,STATE1 | |
| VX STATE3,STATE3,STATE0 | |
| VERLLF STATE3,STATE3,8 | |
| /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */ | |
| VAF STATE2,STATE2,STATE3 | |
| VX STATE1,STATE1,STATE2 | |
| VERLLF STATE1,STATE1,7 | |
| /* STATE1[0,1,2,3] = STATE1[3,0,1,2] */ | |
| VSLDB STATE1,STATE1,STATE1,12 | |
| /* STATE2[0,1,2,3] = STATE2[2,3,0,1] */ | |
| VSLDB STATE2,STATE2,STATE2,8 | |
| /* STATE3[0,1,2,3] = STATE3[1,2,3,0] */ | |
| VSLDB STATE3,STATE3,STATE3,4 | |
| brctg %r0,.Ldoubleround | |
| /* OUTPUT0 = STATE0 + COPY0 */ | |
| VAF STATE0,STATE0,COPY0 | |
| /* OUTPUT1 = STATE1 + COPY1 */ | |
| VAF STATE1,STATE1,COPY1 | |
| /* OUTPUT2 = STATE2 + COPY2 */ | |
| VAF STATE2,STATE2,COPY2 | |
| /* OUTPUT3 = STATE3 + COPY3 */ | |
| VAF STATE3,STATE3,COPY3 | |
| ALTERNATIVE \ | |
| __stringify( \ | |
| /* Convert STATE to little endian and store to OUTPUT */\ | |
| VPERM TMP0,STATE0,STATE0,BEPERM; \ | |
| VPERM TMP1,STATE1,STATE1,BEPERM; \ | |
| VPERM TMP2,STATE2,STATE2,BEPERM; \ | |
| VPERM TMP3,STATE3,STATE3,BEPERM; \ | |
| VSTM TMP0,TMP3,0,%r2), \ | |
| __stringify( \ | |
| /* 32 bit wise little endian store to OUTPUT */ \ | |
| VSTBRF STATE0,0,,%r2; \ | |
| VSTBRF STATE1,16,,%r2; \ | |
| VSTBRF STATE2,32,,%r2; \ | |
| VSTBRF STATE3,48,,%r2; \ | |
| brcl 0,0), \ | |
| ALT_FACILITY(148) | |
| /* ++COPY3.COUNTER */ | |
| /* alsih %r3,1 */ | |
| .insn rilu,0xcc0a00000000,%r3,1 | |
| alcr %r3,%r1 | |
| VLVGG COPY3,%r3,0 | |
| /* OUTPUT += 64, --NBLOCKS */ | |
| aghi %r2,64 | |
| brctg %r5,.Lblock | |
| /* COUNTER = COPY3.COUNTER */ | |
| stg %r3,0(%r4) | |
| /* Zero out potentially sensitive regs */ | |
| VZERO STATE0 | |
| VZERO STATE1 | |
| VZERO STATE2 | |
| VZERO STATE3 | |
| VZERO COPY1 | |
| VZERO COPY2 | |
| /* Early exit if TMP0-TMP3 have not been used */ | |
| ALTERNATIVE "nopr", "br %r14", ALT_FACILITY(148) | |
| VZERO TMP0 | |
| VZERO TMP1 | |
| VZERO TMP2 | |
| VZERO TMP3 | |
| br %r14 | |
| CFI_ENDPROC | |
| SYM_FUNC_END(__arch_chacha20_blocks_nostack) |