| /* SPDX-License-Identifier: GPL-2.0-or-later */ |
| /* |
| * This is a SIMD SHA-1 implementation. It requires the Intel(R) Supplemental |
| * SSE3 instruction set extensions introduced in Intel Core Microarchitecture |
| * processors. CPUs supporting Intel(R) AVX extensions will get an additional |
| * boost. |
| * |
| * This work was inspired by the vectorized implementation of Dean Gaudet. |
| * Additional information on it can be found at: |
| * http://www.arctic.org/~dean/crypto/sha1.html |
| * |
| * It was improved upon with more efficient vectorization of the message |
| * scheduling. This implementation has also been optimized for all current and |
| * several future generations of Intel CPUs. |
| * |
| * See this article for more information about the implementation details: |
| * http://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1/ |
| * |
| * Copyright (C) 2010, Intel Corp. |
| * Authors: Maxim Locktyukhin <maxim.locktyukhin@intel.com> |
| * Ronen Zohar <ronen.zohar@intel.com> |
| * |
| * Converted to AT&T syntax and adapted for inclusion in the Linux kernel: |
| * Author: Mathias Krause <minipli@googlemail.com> |
| */ |
| |
| #include <linux/linkage.h> |
| |
| #define CTX %rdi // arg1 |
| #define BUF %rsi // arg2 |
| #define CNT %rdx // arg3 |
| |
| #define REG_A %ecx |
| #define REG_B %esi |
| #define REG_C %edi |
| #define REG_D %r12d |
| #define REG_E %edx |
| |
| #define REG_T1 %eax |
| #define REG_T2 %ebx |
| |
| #define K_BASE %r8 |
| #define HASH_PTR %r9 |
| #define BUFFER_PTR %r10 |
| #define BUFFER_END %r11 |
| |
| #define W_TMP1 %xmm0 |
| #define W_TMP2 %xmm9 |
| |
| #define W0 %xmm1 |
| #define W4 %xmm2 |
| #define W8 %xmm3 |
| #define W12 %xmm4 |
| #define W16 %xmm5 |
| #define W20 %xmm6 |
| #define W24 %xmm7 |
| #define W28 %xmm8 |
| |
| #define XMM_SHUFB_BSWAP %xmm10 |
| |
| /* we keep window of 64 w[i]+K pre-calculated values in a circular buffer */ |
| #define WK(t) (((t) & 15) * 4)(%rsp) |
| #define W_PRECALC_AHEAD 16 |
| |
| /* |
| * This macro implements the SHA-1 function's body for single 64-byte block |
| * param: function's name |
| */ |
| .macro SHA1_VECTOR_ASM name |
| SYM_FUNC_START(\name) |
| |
| push %rbx |
| push %r12 |
| push %rbp |
| mov %rsp, %rbp |
| |
| sub $64, %rsp # allocate workspace |
| and $~15, %rsp # align stack |
| |
| mov CTX, HASH_PTR |
| mov BUF, BUFFER_PTR |
| |
| shl $6, CNT # multiply by 64 |
| add BUF, CNT |
| mov CNT, BUFFER_END |
| |
| lea K_XMM_AR(%rip), K_BASE |
| xmm_mov BSWAP_SHUFB_CTL(%rip), XMM_SHUFB_BSWAP |
| |
| SHA1_PIPELINED_MAIN_BODY |
| |
| # cleanup workspace |
| mov $8, %ecx |
| mov %rsp, %rdi |
| xor %eax, %eax |
| rep stosq |
| |
| mov %rbp, %rsp # deallocate workspace |
| pop %rbp |
| pop %r12 |
| pop %rbx |
| ret |
| |
| SYM_FUNC_END(\name) |
| .endm |
| |
| /* |
| * This macro implements 80 rounds of SHA-1 for one 64-byte block |
| */ |
| .macro SHA1_PIPELINED_MAIN_BODY |
| INIT_REGALLOC |
| |
| mov (HASH_PTR), A |
| mov 4(HASH_PTR), B |
| mov 8(HASH_PTR), C |
| mov 12(HASH_PTR), D |
| mov 16(HASH_PTR), E |
| |
| .set i, 0 |
| .rept W_PRECALC_AHEAD |
| W_PRECALC i |
| .set i, (i+1) |
| .endr |
| |
| .align 4 |
| 1: |
| RR F1,A,B,C,D,E,0 |
| RR F1,D,E,A,B,C,2 |
| RR F1,B,C,D,E,A,4 |
| RR F1,E,A,B,C,D,6 |
| RR F1,C,D,E,A,B,8 |
| |
| RR F1,A,B,C,D,E,10 |
| RR F1,D,E,A,B,C,12 |
| RR F1,B,C,D,E,A,14 |
| RR F1,E,A,B,C,D,16 |
| RR F1,C,D,E,A,B,18 |
| |
| RR F2,A,B,C,D,E,20 |
| RR F2,D,E,A,B,C,22 |
| RR F2,B,C,D,E,A,24 |
| RR F2,E,A,B,C,D,26 |
| RR F2,C,D,E,A,B,28 |
| |
| RR F2,A,B,C,D,E,30 |
| RR F2,D,E,A,B,C,32 |
| RR F2,B,C,D,E,A,34 |
| RR F2,E,A,B,C,D,36 |
| RR F2,C,D,E,A,B,38 |
| |
| RR F3,A,B,C,D,E,40 |
| RR F3,D,E,A,B,C,42 |
| RR F3,B,C,D,E,A,44 |
| RR F3,E,A,B,C,D,46 |
| RR F3,C,D,E,A,B,48 |
| |
| RR F3,A,B,C,D,E,50 |
| RR F3,D,E,A,B,C,52 |
| RR F3,B,C,D,E,A,54 |
| RR F3,E,A,B,C,D,56 |
| RR F3,C,D,E,A,B,58 |
| |
| add $64, BUFFER_PTR # move to the next 64-byte block |
| cmp BUFFER_END, BUFFER_PTR # if the current is the last one use |
| cmovae K_BASE, BUFFER_PTR # dummy source to avoid buffer overrun |
| |
| RR F4,A,B,C,D,E,60 |
| RR F4,D,E,A,B,C,62 |
| RR F4,B,C,D,E,A,64 |
| RR F4,E,A,B,C,D,66 |
| RR F4,C,D,E,A,B,68 |
| |
| RR F4,A,B,C,D,E,70 |
| RR F4,D,E,A,B,C,72 |
| RR F4,B,C,D,E,A,74 |
| RR F4,E,A,B,C,D,76 |
| RR F4,C,D,E,A,B,78 |
| |
| UPDATE_HASH (HASH_PTR), A |
| UPDATE_HASH 4(HASH_PTR), B |
| UPDATE_HASH 8(HASH_PTR), C |
| UPDATE_HASH 12(HASH_PTR), D |
| UPDATE_HASH 16(HASH_PTR), E |
| |
| RESTORE_RENAMED_REGS |
| cmp K_BASE, BUFFER_PTR # K_BASE means, we reached the end |
| jne 1b |
| .endm |
| |
| .macro INIT_REGALLOC |
| .set A, REG_A |
| .set B, REG_B |
| .set C, REG_C |
| .set D, REG_D |
| .set E, REG_E |
| .set T1, REG_T1 |
| .set T2, REG_T2 |
| .endm |
| |
| .macro RESTORE_RENAMED_REGS |
| # order is important (REG_C is where it should be) |
| mov B, REG_B |
| mov D, REG_D |
| mov A, REG_A |
| mov E, REG_E |
| .endm |
| |
| .macro SWAP_REG_NAMES a, b |
| .set _T, \a |
| .set \a, \b |
| .set \b, _T |
| .endm |
| |
| .macro F1 b, c, d |
| mov \c, T1 |
| SWAP_REG_NAMES \c, T1 |
| xor \d, T1 |
| and \b, T1 |
| xor \d, T1 |
| .endm |
| |
| .macro F2 b, c, d |
| mov \d, T1 |
| SWAP_REG_NAMES \d, T1 |
| xor \c, T1 |
| xor \b, T1 |
| .endm |
| |
| .macro F3 b, c ,d |
| mov \c, T1 |
| SWAP_REG_NAMES \c, T1 |
| mov \b, T2 |
| or \b, T1 |
| and \c, T2 |
| and \d, T1 |
| or T2, T1 |
| .endm |
| |
| .macro F4 b, c, d |
| F2 \b, \c, \d |
| .endm |
| |
| .macro UPDATE_HASH hash, val |
| add \hash, \val |
| mov \val, \hash |
| .endm |
| |
| /* |
| * RR does two rounds of SHA-1 back to back with W[] pre-calc |
| * t1 = F(b, c, d); e += w(i) |
| * e += t1; b <<= 30; d += w(i+1); |
| * t1 = F(a, b, c); |
| * d += t1; a <<= 5; |
| * e += a; |
| * t1 = e; a >>= 7; |
| * t1 <<= 5; |
| * d += t1; |
| */ |
| .macro RR F, a, b, c, d, e, round |
| add WK(\round), \e |
| \F \b, \c, \d # t1 = F(b, c, d); |
| W_PRECALC (\round + W_PRECALC_AHEAD) |
| rol $30, \b |
| add T1, \e |
| add WK(\round + 1), \d |
| |
| \F \a, \b, \c |
| W_PRECALC (\round + W_PRECALC_AHEAD + 1) |
| rol $5, \a |
| add \a, \e |
| add T1, \d |
| ror $7, \a # (a <<r 5) >>r 7) => a <<r 30) |
| |
| mov \e, T1 |
| SWAP_REG_NAMES \e, T1 |
| |
| rol $5, T1 |
| add T1, \d |
| |
| # write: \a, \b |
| # rotate: \a<=\d, \b<=\e, \c<=\a, \d<=\b, \e<=\c |
| .endm |
| |
| .macro W_PRECALC r |
| .set i, \r |
| |
| .if (i < 20) |
| .set K_XMM, 0 |
| .elseif (i < 40) |
| .set K_XMM, 16 |
| .elseif (i < 60) |
| .set K_XMM, 32 |
| .elseif (i < 80) |
| .set K_XMM, 48 |
| .endif |
| |
| .if ((i < 16) || ((i >= 80) && (i < (80 + W_PRECALC_AHEAD)))) |
| .set i, ((\r) % 80) # pre-compute for the next iteration |
| .if (i == 0) |
| W_PRECALC_RESET |
| .endif |
| W_PRECALC_00_15 |
| .elseif (i<32) |
| W_PRECALC_16_31 |
| .elseif (i < 80) // rounds 32-79 |
| W_PRECALC_32_79 |
| .endif |
| .endm |
| |
| .macro W_PRECALC_RESET |
| .set W, W0 |
| .set W_minus_04, W4 |
| .set W_minus_08, W8 |
| .set W_minus_12, W12 |
| .set W_minus_16, W16 |
| .set W_minus_20, W20 |
| .set W_minus_24, W24 |
| .set W_minus_28, W28 |
| .set W_minus_32, W |
| .endm |
| |
| .macro W_PRECALC_ROTATE |
| .set W_minus_32, W_minus_28 |
| .set W_minus_28, W_minus_24 |
| .set W_minus_24, W_minus_20 |
| .set W_minus_20, W_minus_16 |
| .set W_minus_16, W_minus_12 |
| .set W_minus_12, W_minus_08 |
| .set W_minus_08, W_minus_04 |
| .set W_minus_04, W |
| .set W, W_minus_32 |
| .endm |
| |
| .macro W_PRECALC_SSSE3 |
| |
| .macro W_PRECALC_00_15 |
| W_PRECALC_00_15_SSSE3 |
| .endm |
| .macro W_PRECALC_16_31 |
| W_PRECALC_16_31_SSSE3 |
| .endm |
| .macro W_PRECALC_32_79 |
| W_PRECALC_32_79_SSSE3 |
| .endm |
| |
| /* message scheduling pre-compute for rounds 0-15 */ |
| .macro W_PRECALC_00_15_SSSE3 |
| .if ((i & 3) == 0) |
| movdqu (i*4)(BUFFER_PTR), W_TMP1 |
| .elseif ((i & 3) == 1) |
| pshufb XMM_SHUFB_BSWAP, W_TMP1 |
| movdqa W_TMP1, W |
| .elseif ((i & 3) == 2) |
| paddd (K_BASE), W_TMP1 |
| .elseif ((i & 3) == 3) |
| movdqa W_TMP1, WK(i&~3) |
| W_PRECALC_ROTATE |
| .endif |
| .endm |
| |
| /* message scheduling pre-compute for rounds 16-31 |
| * |
| * - calculating last 32 w[i] values in 8 XMM registers |
| * - pre-calculate K+w[i] values and store to mem, for later load by ALU add |
| * instruction |
| * |
| * some "heavy-lifting" vectorization for rounds 16-31 due to w[i]->w[i-3] |
| * dependency, but improves for 32-79 |
| */ |
| .macro W_PRECALC_16_31_SSSE3 |
| # blended scheduling of vector and scalar instruction streams, one 4-wide |
| # vector iteration / 4 scalar rounds |
| .if ((i & 3) == 0) |
| movdqa W_minus_12, W |
| palignr $8, W_minus_16, W # w[i-14] |
| movdqa W_minus_04, W_TMP1 |
| psrldq $4, W_TMP1 # w[i-3] |
| pxor W_minus_08, W |
| .elseif ((i & 3) == 1) |
| pxor W_minus_16, W_TMP1 |
| pxor W_TMP1, W |
| movdqa W, W_TMP2 |
| movdqa W, W_TMP1 |
| pslldq $12, W_TMP2 |
| .elseif ((i & 3) == 2) |
| psrld $31, W |
| pslld $1, W_TMP1 |
| por W, W_TMP1 |
| movdqa W_TMP2, W |
| psrld $30, W_TMP2 |
| pslld $2, W |
| .elseif ((i & 3) == 3) |
| pxor W, W_TMP1 |
| pxor W_TMP2, W_TMP1 |
| movdqa W_TMP1, W |
| paddd K_XMM(K_BASE), W_TMP1 |
| movdqa W_TMP1, WK(i&~3) |
| W_PRECALC_ROTATE |
| .endif |
| .endm |
| |
| /* message scheduling pre-compute for rounds 32-79 |
| * |
| * in SHA-1 specification: w[i] = (w[i-3] ^ w[i-8] ^ w[i-14] ^ w[i-16]) rol 1 |
| * instead we do equal: w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2 |
| * allows more efficient vectorization since w[i]=>w[i-3] dependency is broken |
| */ |
| .macro W_PRECALC_32_79_SSSE3 |
| .if ((i & 3) == 0) |
| movdqa W_minus_04, W_TMP1 |
| pxor W_minus_28, W # W is W_minus_32 before xor |
| palignr $8, W_minus_08, W_TMP1 |
| .elseif ((i & 3) == 1) |
| pxor W_minus_16, W |
| pxor W_TMP1, W |
| movdqa W, W_TMP1 |
| .elseif ((i & 3) == 2) |
| psrld $30, W |
| pslld $2, W_TMP1 |
| por W, W_TMP1 |
| .elseif ((i & 3) == 3) |
| movdqa W_TMP1, W |
| paddd K_XMM(K_BASE), W_TMP1 |
| movdqa W_TMP1, WK(i&~3) |
| W_PRECALC_ROTATE |
| .endif |
| .endm |
| |
| .endm // W_PRECALC_SSSE3 |
| |
| |
| #define K1 0x5a827999 |
| #define K2 0x6ed9eba1 |
| #define K3 0x8f1bbcdc |
| #define K4 0xca62c1d6 |
| |
| .section .rodata |
| .align 16 |
| |
| K_XMM_AR: |
| .long K1, K1, K1, K1 |
| .long K2, K2, K2, K2 |
| .long K3, K3, K3, K3 |
| .long K4, K4, K4, K4 |
| |
| BSWAP_SHUFB_CTL: |
| .long 0x00010203 |
| .long 0x04050607 |
| .long 0x08090a0b |
| .long 0x0c0d0e0f |
| |
| |
| .section .text |
| |
| W_PRECALC_SSSE3 |
| .macro xmm_mov a, b |
| movdqu \a,\b |
| .endm |
| |
| /* |
| * SSSE3 optimized implementation: |
| * |
| * extern "C" void sha1_transform_ssse3(struct sha1_state *state, |
| * const u8 *data, int blocks); |
| * |
| * Note that struct sha1_state is assumed to begin with u32 state[5]. |
| */ |
| SHA1_VECTOR_ASM sha1_transform_ssse3 |
| |
| #ifdef CONFIG_AS_AVX |
| |
| .macro W_PRECALC_AVX |
| |
| .purgem W_PRECALC_00_15 |
| .macro W_PRECALC_00_15 |
| W_PRECALC_00_15_AVX |
| .endm |
| .purgem W_PRECALC_16_31 |
| .macro W_PRECALC_16_31 |
| W_PRECALC_16_31_AVX |
| .endm |
| .purgem W_PRECALC_32_79 |
| .macro W_PRECALC_32_79 |
| W_PRECALC_32_79_AVX |
| .endm |
| |
| .macro W_PRECALC_00_15_AVX |
| .if ((i & 3) == 0) |
| vmovdqu (i*4)(BUFFER_PTR), W_TMP1 |
| .elseif ((i & 3) == 1) |
| vpshufb XMM_SHUFB_BSWAP, W_TMP1, W |
| .elseif ((i & 3) == 2) |
| vpaddd (K_BASE), W, W_TMP1 |
| .elseif ((i & 3) == 3) |
| vmovdqa W_TMP1, WK(i&~3) |
| W_PRECALC_ROTATE |
| .endif |
| .endm |
| |
| .macro W_PRECALC_16_31_AVX |
| .if ((i & 3) == 0) |
| vpalignr $8, W_minus_16, W_minus_12, W # w[i-14] |
| vpsrldq $4, W_minus_04, W_TMP1 # w[i-3] |
| vpxor W_minus_08, W, W |
| vpxor W_minus_16, W_TMP1, W_TMP1 |
| .elseif ((i & 3) == 1) |
| vpxor W_TMP1, W, W |
| vpslldq $12, W, W_TMP2 |
| vpslld $1, W, W_TMP1 |
| .elseif ((i & 3) == 2) |
| vpsrld $31, W, W |
| vpor W, W_TMP1, W_TMP1 |
| vpslld $2, W_TMP2, W |
| vpsrld $30, W_TMP2, W_TMP2 |
| .elseif ((i & 3) == 3) |
| vpxor W, W_TMP1, W_TMP1 |
| vpxor W_TMP2, W_TMP1, W |
| vpaddd K_XMM(K_BASE), W, W_TMP1 |
| vmovdqu W_TMP1, WK(i&~3) |
| W_PRECALC_ROTATE |
| .endif |
| .endm |
| |
| .macro W_PRECALC_32_79_AVX |
| .if ((i & 3) == 0) |
| vpalignr $8, W_minus_08, W_minus_04, W_TMP1 |
| vpxor W_minus_28, W, W # W is W_minus_32 before xor |
| .elseif ((i & 3) == 1) |
| vpxor W_minus_16, W_TMP1, W_TMP1 |
| vpxor W_TMP1, W, W |
| .elseif ((i & 3) == 2) |
| vpslld $2, W, W_TMP1 |
| vpsrld $30, W, W |
| vpor W, W_TMP1, W |
| .elseif ((i & 3) == 3) |
| vpaddd K_XMM(K_BASE), W, W_TMP1 |
| vmovdqu W_TMP1, WK(i&~3) |
| W_PRECALC_ROTATE |
| .endif |
| .endm |
| |
| .endm // W_PRECALC_AVX |
| |
| W_PRECALC_AVX |
| .purgem xmm_mov |
| .macro xmm_mov a, b |
| vmovdqu \a,\b |
| .endm |
| |
| |
| /* AVX optimized implementation: |
| * extern "C" void sha1_transform_avx(struct sha1_state *state, |
| * const u8 *data, int blocks); |
| */ |
| SHA1_VECTOR_ASM sha1_transform_avx |
| |
| #endif |