| /* SPDX-License-Identifier: GPL-2.0-or-later */ |
| # |
| # Accelerated AES-GCM stitched implementation for ppc64le. |
| # |
| # Copyright 2024- IBM Inc. |
| # |
| #=================================================================================== |
| # Written by Danny Tsen <dtsen@us.ibm.com> |
| # |
| # GHASH is based on the Karatsuba multiplication method. |
| # |
| # Xi xor X1 |
| # |
| # X1 * H^4 + X2 * H^3 + x3 * H^2 + X4 * H = |
| # (X1.h * H4.h + xX.l * H4.l + X1 * H4) + |
| # (X2.h * H3.h + X2.l * H3.l + X2 * H3) + |
| # (X3.h * H2.h + X3.l * H2.l + X3 * H2) + |
| # (X4.h * H.h + X4.l * H.l + X4 * H) |
| # |
| # Xi = v0 |
| # H Poly = v2 |
| # Hash keys = v3 - v14 |
| # ( H.l, H, H.h) |
| # ( H^2.l, H^2, H^2.h) |
| # ( H^3.l, H^3, H^3.h) |
| # ( H^4.l, H^4, H^4.h) |
| # |
| # v30 is IV |
| # v31 - counter 1 |
| # |
| # AES used, |
| # vs0 - round key 0 |
| # v15, v16, v17, v18, v19, v20, v21, v22 for 8 blocks (encrypted) |
| # |
| # This implementation uses stitched AES-GCM approach to improve overall performance. |
| # AES is implemented with 8x blocks and GHASH is using 2 4x blocks. |
| # |
| # =================================================================================== |
| # |
| |
| #include <asm/ppc_asm.h> |
| #include <linux/linkage.h> |
| |
| .machine "any" |
| .text |
| |
| .macro SAVE_GPR GPR OFFSET FRAME |
| std \GPR,\OFFSET(\FRAME) |
| .endm |
| |
| .macro SAVE_VRS VRS OFFSET FRAME |
| stxv \VRS+32, \OFFSET(\FRAME) |
| .endm |
| |
| .macro RESTORE_GPR GPR OFFSET FRAME |
| ld \GPR,\OFFSET(\FRAME) |
| .endm |
| |
| .macro RESTORE_VRS VRS OFFSET FRAME |
| lxv \VRS+32, \OFFSET(\FRAME) |
| .endm |
| |
| .macro SAVE_REGS |
| mflr 0 |
| std 0, 16(1) |
| stdu 1,-512(1) |
| |
| SAVE_GPR 14, 112, 1 |
| SAVE_GPR 15, 120, 1 |
| SAVE_GPR 16, 128, 1 |
| SAVE_GPR 17, 136, 1 |
| SAVE_GPR 18, 144, 1 |
| SAVE_GPR 19, 152, 1 |
| SAVE_GPR 20, 160, 1 |
| SAVE_GPR 21, 168, 1 |
| SAVE_GPR 22, 176, 1 |
| SAVE_GPR 23, 184, 1 |
| SAVE_GPR 24, 192, 1 |
| |
| addi 9, 1, 256 |
| SAVE_VRS 20, 0, 9 |
| SAVE_VRS 21, 16, 9 |
| SAVE_VRS 22, 32, 9 |
| SAVE_VRS 23, 48, 9 |
| SAVE_VRS 24, 64, 9 |
| SAVE_VRS 25, 80, 9 |
| SAVE_VRS 26, 96, 9 |
| SAVE_VRS 27, 112, 9 |
| SAVE_VRS 28, 128, 9 |
| SAVE_VRS 29, 144, 9 |
| SAVE_VRS 30, 160, 9 |
| SAVE_VRS 31, 176, 9 |
| .endm # SAVE_REGS |
| |
| .macro RESTORE_REGS |
| addi 9, 1, 256 |
| RESTORE_VRS 20, 0, 9 |
| RESTORE_VRS 21, 16, 9 |
| RESTORE_VRS 22, 32, 9 |
| RESTORE_VRS 23, 48, 9 |
| RESTORE_VRS 24, 64, 9 |
| RESTORE_VRS 25, 80, 9 |
| RESTORE_VRS 26, 96, 9 |
| RESTORE_VRS 27, 112, 9 |
| RESTORE_VRS 28, 128, 9 |
| RESTORE_VRS 29, 144, 9 |
| RESTORE_VRS 30, 160, 9 |
| RESTORE_VRS 31, 176, 9 |
| |
| RESTORE_GPR 14, 112, 1 |
| RESTORE_GPR 15, 120, 1 |
| RESTORE_GPR 16, 128, 1 |
| RESTORE_GPR 17, 136, 1 |
| RESTORE_GPR 18, 144, 1 |
| RESTORE_GPR 19, 152, 1 |
| RESTORE_GPR 20, 160, 1 |
| RESTORE_GPR 21, 168, 1 |
| RESTORE_GPR 22, 176, 1 |
| RESTORE_GPR 23, 184, 1 |
| RESTORE_GPR 24, 192, 1 |
| |
| addi 1, 1, 512 |
| ld 0, 16(1) |
| mtlr 0 |
| .endm # RESTORE_REGS |
| |
| # 4x loops |
| .macro AES_CIPHER_4x _VCIPHER ST r |
| \_VCIPHER \ST, \ST, \r |
| \_VCIPHER \ST+1, \ST+1, \r |
| \_VCIPHER \ST+2, \ST+2, \r |
| \_VCIPHER \ST+3, \ST+3, \r |
| .endm |
| |
| # 8x loops |
| .macro AES_CIPHER_8x _VCIPHER ST r |
| \_VCIPHER \ST, \ST, \r |
| \_VCIPHER \ST+1, \ST+1, \r |
| \_VCIPHER \ST+2, \ST+2, \r |
| \_VCIPHER \ST+3, \ST+3, \r |
| \_VCIPHER \ST+4, \ST+4, \r |
| \_VCIPHER \ST+5, \ST+5, \r |
| \_VCIPHER \ST+6, \ST+6, \r |
| \_VCIPHER \ST+7, \ST+7, \r |
| .endm |
| |
| .macro LOOP_8AES_STATE |
| xxlor 32+23, 1, 1 |
| xxlor 32+24, 2, 2 |
| xxlor 32+25, 3, 3 |
| xxlor 32+26, 4, 4 |
| AES_CIPHER_8x vcipher, 15, 23 |
| AES_CIPHER_8x vcipher, 15, 24 |
| AES_CIPHER_8x vcipher, 15, 25 |
| AES_CIPHER_8x vcipher, 15, 26 |
| xxlor 32+23, 5, 5 |
| xxlor 32+24, 6, 6 |
| xxlor 32+25, 7, 7 |
| xxlor 32+26, 8, 8 |
| AES_CIPHER_8x vcipher, 15, 23 |
| AES_CIPHER_8x vcipher, 15, 24 |
| AES_CIPHER_8x vcipher, 15, 25 |
| AES_CIPHER_8x vcipher, 15, 26 |
| .endm |
| |
| # |
| # PPC_GHASH4x(H, S1, S2, S3, S4): Compute 4x hash values based on Karatsuba method. |
| # H: returning digest |
| # S#: states |
| # |
| # S1 should xor with the previous digest |
| # |
| # Xi = v0 |
| # H Poly = v2 |
| # Hash keys = v3 - v14 |
| # Scratch: v23 - v29 |
| # |
| .macro PPC_GHASH4x H S1 S2 S3 S4 |
| |
| vpmsumd 23, 12, \S1 # H4.L * X.L |
| vpmsumd 24, 9, \S2 |
| vpmsumd 25, 6, \S3 |
| vpmsumd 26, 3, \S4 |
| |
| vpmsumd 27, 13, \S1 # H4.L * X.H + H4.H * X.L |
| vpmsumd 28, 10, \S2 # H3.L * X1.H + H3.H * X1.L |
| |
| vxor 23, 23, 24 |
| vxor 23, 23, 25 |
| vxor 23, 23, 26 # L |
| |
| vxor 24, 27, 28 |
| vpmsumd 25, 7, \S3 |
| vpmsumd 26, 4, \S4 |
| |
| vxor 24, 24, 25 |
| vxor 24, 24, 26 # M |
| |
| # sum hash and reduction with H Poly |
| vpmsumd 28, 23, 2 # reduction |
| |
| vxor 1, 1, 1 |
| vsldoi 25, 24, 1, 8 # mL |
| vsldoi 1, 1, 24, 8 # mH |
| vxor 23, 23, 25 # mL + L |
| |
| # This performs swap and xor like, |
| # vsldoi 23, 23, 23, 8 # swap |
| # vxor 23, 23, 28 |
| xxlor 32+25, 10, 10 |
| vpermxor 23, 23, 28, 25 |
| |
| vpmsumd 26, 14, \S1 # H4.H * X.H |
| vpmsumd 27, 11, \S2 |
| vpmsumd 28, 8, \S3 |
| vpmsumd 29, 5, \S4 |
| |
| vxor 24, 26, 27 |
| vxor 24, 24, 28 |
| vxor 24, 24, 29 |
| |
| vxor 24, 24, 1 |
| |
| # sum hash and reduction with H Poly |
| vsldoi 25, 23, 23, 8 # swap |
| vpmsumd 23, 23, 2 |
| vxor 27, 25, 24 |
| vxor \H, 23, 27 |
| .endm |
| |
| # |
| # Compute update single ghash |
| # scratch: v1, v22..v27 |
| # |
| .macro PPC_GHASH1x H S1 |
| |
| vxor 1, 1, 1 |
| |
| vpmsumd 22, 3, \S1 # L |
| vpmsumd 23, 4, \S1 # M |
| vpmsumd 24, 5, \S1 # H |
| |
| vpmsumd 27, 22, 2 # reduction |
| |
| vsldoi 25, 23, 1, 8 # mL |
| vsldoi 26, 1, 23, 8 # mH |
| vxor 22, 22, 25 # LL + LL |
| vxor 24, 24, 26 # HH + HH |
| |
| xxlor 32+25, 10, 10 |
| vpermxor 22, 22, 27, 25 |
| |
| vsldoi 23, 22, 22, 8 # swap |
| vpmsumd 22, 22, 2 # reduction |
| vxor 23, 23, 24 |
| vxor \H, 22, 23 |
| .endm |
| |
| # |
| # LOAD_HASH_TABLE |
| # Xi = v0 |
| # H Poly = v2 |
| # Hash keys = v3 - v14 |
| # |
| .macro LOAD_HASH_TABLE |
| # Load Xi |
| lxvb16x 32, 0, 8 # load Xi |
| |
| # load Hash - h^4, h^3, h^2, h |
| li 10, 32 |
| lxvd2x 2+32, 10, 8 # H Poli |
| li 10, 48 |
| lxvd2x 3+32, 10, 8 # Hl |
| li 10, 64 |
| lxvd2x 4+32, 10, 8 # H |
| li 10, 80 |
| lxvd2x 5+32, 10, 8 # Hh |
| |
| li 10, 96 |
| lxvd2x 6+32, 10, 8 # H^2l |
| li 10, 112 |
| lxvd2x 7+32, 10, 8 # H^2 |
| li 10, 128 |
| lxvd2x 8+32, 10, 8 # H^2h |
| |
| li 10, 144 |
| lxvd2x 9+32, 10, 8 # H^3l |
| li 10, 160 |
| lxvd2x 10+32, 10, 8 # H^3 |
| li 10, 176 |
| lxvd2x 11+32, 10, 8 # H^3h |
| |
| li 10, 192 |
| lxvd2x 12+32, 10, 8 # H^4l |
| li 10, 208 |
| lxvd2x 13+32, 10, 8 # H^4 |
| li 10, 224 |
| lxvd2x 14+32, 10, 8 # H^4h |
| .endm |
| |
| ################################################################################ |
| # Compute AES and ghash one block at a time. |
| # r23: AES rounds |
| # v30: current IV |
| # vs0: roundkey 0 |
| # |
| ################################################################################ |
| SYM_FUNC_START_LOCAL(aes_gcm_crypt_1x) |
| |
| cmpdi 5, 16 |
| bge __More_1x |
| blr |
| __More_1x: |
| li 10, 16 |
| divdu 12, 5, 10 |
| |
| xxlxor 32+15, 32+30, 0 |
| |
| # Pre-load 8 AES rounds to scratch vectors. |
| xxlor 32+16, 1, 1 |
| xxlor 32+17, 2, 2 |
| xxlor 32+18, 3, 3 |
| xxlor 32+19, 4, 4 |
| xxlor 32+20, 5, 5 |
| xxlor 32+21, 6, 6 |
| xxlor 32+28, 7, 7 |
| xxlor 32+29, 8, 8 |
| lwz 23, 240(6) # n rounds |
| addi 22, 23, -9 # remaing AES rounds |
| |
| cmpdi 12, 0 |
| bgt __Loop_1x |
| blr |
| |
| __Loop_1x: |
| mtctr 22 |
| addi 10, 6, 144 |
| vcipher 15, 15, 16 |
| vcipher 15, 15, 17 |
| vcipher 15, 15, 18 |
| vcipher 15, 15, 19 |
| vcipher 15, 15, 20 |
| vcipher 15, 15, 21 |
| vcipher 15, 15, 28 |
| vcipher 15, 15, 29 |
| |
| __Loop_aes_1state: |
| lxv 32+1, 0(10) |
| vcipher 15, 15, 1 |
| addi 10, 10, 16 |
| bdnz __Loop_aes_1state |
| lxv 32+1, 0(10) # last round key |
| lxvb16x 11, 0, 14 # load input block |
| vcipherlast 15, 15, 1 |
| |
| xxlxor 32+15, 32+15, 11 |
| stxvb16x 32+15, 0, 9 # store output |
| addi 14, 14, 16 |
| addi 9, 9, 16 |
| |
| cmpdi 24, 0 # decrypt? |
| bne __Encrypt_1x |
| xxlor 15+32, 11, 11 |
| __Encrypt_1x: |
| vxor 15, 15, 0 |
| PPC_GHASH1x 0, 15 |
| |
| addi 5, 5, -16 |
| addi 11, 11, 16 |
| |
| vadduwm 30, 30, 31 # IV + counter |
| xxlxor 32+15, 32+30, 0 |
| addi 12, 12, -1 |
| cmpdi 12, 0 |
| bgt __Loop_1x |
| |
| stxvb16x 32+30, 0, 7 # update IV |
| stxvb16x 32+0, 0, 8 # update Xi |
| blr |
| SYM_FUNC_END(aes_gcm_crypt_1x) |
| |
| ################################################################################ |
| # Process a normal partial block when we come here. |
| # Compute partial mask, Load and store partial block to stack. |
| # Update partial_len and pblock. |
| # pblock is (encrypted ^ AES state) for encrypt |
| # and (input ^ AES state) for decrypt. |
| # |
| ################################################################################ |
| SYM_FUNC_START_LOCAL(__Process_partial) |
| |
| # create partial mask |
| vspltisb 16, -1 |
| li 12, 16 |
| sub 12, 12, 5 |
| sldi 12, 12, 3 |
| mtvsrdd 32+17, 0, 12 |
| vslo 16, 16, 17 # partial block mask |
| |
| lxvb16x 11, 0, 14 # load partial block |
| xxland 11, 11, 32+16 |
| |
| # AES crypt partial |
| xxlxor 32+15, 32+30, 0 |
| lwz 23, 240(6) # n rounds |
| addi 22, 23, -1 # loop - 1 |
| mtctr 22 |
| addi 10, 6, 16 |
| |
| __Loop_aes_pstate: |
| lxv 32+1, 0(10) |
| vcipher 15, 15, 1 |
| addi 10, 10, 16 |
| bdnz __Loop_aes_pstate |
| lxv 32+1, 0(10) # last round key |
| vcipherlast 15, 15, 1 |
| |
| xxlxor 32+15, 32+15, 11 |
| vand 15, 15, 16 |
| |
| # AES crypt output v15 |
| # Write partial |
| li 10, 224 |
| stxvb16x 15+32, 10, 1 # write v15 to stack |
| addi 10, 1, 223 |
| addi 12, 9, -1 |
| mtctr 5 # partial block len |
| __Write_partial: |
| lbzu 22, 1(10) |
| stbu 22, 1(12) |
| bdnz __Write_partial |
| |
| cmpdi 24, 0 # decrypt? |
| bne __Encrypt_partial |
| xxlor 32+15, 11, 11 # decrypt using the input block |
| __Encrypt_partial: |
| #vxor 15, 15, 0 # ^ previous hash |
| #PPC_GHASH1x 0, 15 |
| |
| add 14, 14, 5 |
| add 9, 9, 5 |
| std 5, 56(7) # update partial |
| sub 11, 11, 5 |
| li 5, 0 # done last byte |
| |
| # |
| # Don't increase IV since this is the last partial. |
| # It should get updated in gcm_update if no more data blocks. |
| #vadduwm 30, 30, 31 # increase IV |
| stxvb16x 32+30, 0, 7 # update IV |
| li 10, 64 |
| stxvb16x 32+0, 0, 8 # Update X1 |
| stxvb16x 32+15, 10, 7 # Update pblock |
| blr |
| SYM_FUNC_END(__Process_partial) |
| |
| ################################################################################ |
| # Combine partial blocks and ghash when we come here. |
| # |
| # The partial block has to be shifted to the right location to encrypt/decrypt |
| # and compute ghash if combing the previous partial block is needed. |
| # - Compute ghash for a full block. Clear Partial_len and pblock. Update IV. |
| # Write Xi. |
| # - Don't compute ghash if not full block. gcm_update will take care of it |
| # is the last block. Update Partial_len and pblock. |
| # |
| ################################################################################ |
| SYM_FUNC_START_LOCAL(__Combine_partial) |
| |
| ld 12, 56(7) |
| mr 21, 5 # these bytes to be processed |
| |
| li 17, 0 |
| li 16, 16 |
| sub 22, 16, 12 # bytes to complete a block |
| sub 17, 22, 5 # remaining bytes in a block |
| cmpdi 5, 16 |
| ble __Inp_msg_less16 |
| li 17, 0 |
| mr 21, 22 |
| b __Combine_continue |
| __Inp_msg_less16: |
| cmpd 22, 5 |
| bgt __Combine_continue |
| li 17, 0 |
| mr 21, 22 # these bytes to be processed |
| |
| __Combine_continue: |
| # load msg and shift to the proper location and mask |
| vspltisb 16, -1 |
| sldi 15, 12, 3 |
| mtvsrdd 32+17, 0, 15 |
| vslo 16, 16, 17 |
| vsro 16, 16, 17 |
| sldi 15, 17, 3 |
| mtvsrdd 32+17, 0, 15 |
| vsro 16, 16, 17 |
| vslo 16, 16, 17 # mask |
| |
| lxvb16x 32+19, 0, 14 # load partial block |
| sldi 15, 12, 3 |
| mtvsrdd 32+17, 0, 15 |
| vsro 19, 19, 17 # 0x00..xxxx??..?? |
| sldi 15, 17, 3 |
| mtvsrdd 32+17, 0, 15 |
| vsro 19, 19, 17 # 0x00..xxxx |
| vslo 19, 19, 17 # shift back to form 0x00..xxxx00..00 |
| |
| # AES crypt partial |
| xxlxor 32+15, 32+30, 0 |
| lwz 23, 240(6) # n rounds |
| addi 22, 23, -1 # loop - 1 |
| mtctr 22 |
| addi 10, 6, 16 |
| |
| __Loop_aes_cpstate: |
| lxv 32+1, 0(10) |
| vcipher 15, 15, 1 |
| addi 10, 10, 16 |
| bdnz __Loop_aes_cpstate |
| lxv 32+1, 0(10) # last round key |
| vcipherlast 15, 15, 1 |
| |
| vxor 15, 15, 19 |
| vand 15, 15, 16 |
| |
| # AES crypt output v15 |
| # Write partial |
| li 10, 224 |
| stxvb16x 15+32, 10, 1 # write v15 to stack |
| addi 10, 1, 223 |
| add 10, 10, 12 # add offset |
| addi 15, 9, -1 |
| mtctr 21 # partial block len |
| __Write_combine_partial: |
| lbzu 22, 1(10) |
| stbu 22, 1(15) |
| bdnz __Write_combine_partial |
| |
| add 14, 14, 21 |
| add 11, 11, 21 |
| add 9, 9, 21 |
| sub 5, 5, 21 |
| |
| # Encrypt/Decrypt? |
| cmpdi 24, 0 # decrypt? |
| bne __Encrypt_combine_partial |
| vmr 15, 19 # decrypt using the input block |
| |
| __Encrypt_combine_partial: |
| # |
| # Update partial flag and combine ghash. |
| __Update_partial_ghash: |
| li 10, 64 |
| lxvb16x 32+17, 10, 7 # load previous pblock |
| add 12, 12, 21 # combined pprocessed |
| vxor 15, 15, 17 # combined pblock |
| |
| cmpdi 12, 16 |
| beq __Clear_partial_flag |
| std 12, 56(7) # update partial len |
| stxvb16x 32+15, 10, 7 # Update current pblock |
| blr |
| |
| __Clear_partial_flag: |
| li 12, 0 |
| std 12, 56(7) |
| # Update IV and ghash here |
| vadduwm 30, 30, 31 # increase IV |
| stxvb16x 32+30, 0, 7 # update IV |
| |
| # v15 either is either (input blockor encrypted)^(AES state) |
| vxor 15, 15, 0 |
| PPC_GHASH1x 0, 15 |
| stxvb16x 32+0, 10, 7 # update pblock for debug? |
| stxvb16x 32+0, 0, 8 # update Xi |
| blr |
| SYM_FUNC_END(__Combine_partial) |
| |
| ################################################################################ |
| # gcm_update(iv, Xi) - compute last hash |
| # |
| ################################################################################ |
| SYM_FUNC_START(gcm_update) |
| |
| ld 10, 56(3) |
| cmpdi 10, 0 |
| beq __no_update |
| |
| lxvb16x 32, 0, 4 # load Xi |
| # load Hash - h^4, h^3, h^2, h |
| li 10, 32 |
| lxvd2x 2+32, 10, 4 # H Poli |
| li 10, 48 |
| lxvd2x 3+32, 10, 4 # Hl |
| li 10, 64 |
| lxvd2x 4+32, 10, 4 # H |
| li 10, 80 |
| lxvd2x 5+32, 10, 4 # Hh |
| |
| addis 11, 2, permx@toc@ha |
| addi 11, 11, permx@toc@l |
| lxv 10, 0(11) # vs10: vpermxor vector |
| |
| li 9, 64 |
| lxvb16x 32+6, 9, 3 # load pblock |
| vxor 6, 6, 0 |
| |
| vxor 1, 1, 1 |
| vpmsumd 12, 3, 6 # L |
| vpmsumd 13, 4, 6 # M |
| vpmsumd 14, 5, 6 # H |
| vpmsumd 17, 12, 2 # reduction |
| vsldoi 15, 13, 1, 8 # mL |
| vsldoi 16, 1, 13, 8 # mH |
| vxor 12, 12, 15 # LL + LL |
| vxor 14, 14, 16 # HH + HH |
| xxlor 32+15, 10, 10 |
| vpermxor 12, 12, 17, 15 |
| vsldoi 13, 12, 12, 8 # swap |
| vpmsumd 12, 12, 2 # reduction |
| vxor 13, 13, 14 |
| vxor 7, 12, 13 |
| |
| #vxor 0, 0, 0 |
| #stxvb16x 32+0, 9, 3 |
| li 10, 0 |
| std 10, 56(3) |
| stxvb16x 32+7, 0, 4 |
| |
| __no_update: |
| blr |
| SYM_FUNC_END(gcm_update) |
| |
| ################################################################################ |
| # aes_p10_gcm_encrypt (const void *inp, void *out, size_t len, |
| # const char *rk, unsigned char iv[16], void *Xip); |
| # |
| # r3 - inp |
| # r4 - out |
| # r5 - len |
| # r6 - AES round keys |
| # r7 - iv and other data |
| # r8 - Xi, HPoli, hash keys |
| # |
| # rounds is at offset 240 in rk |
| # Xi is at 0 in gcm_table (Xip). |
| # |
| ################################################################################ |
| SYM_FUNC_START(aes_p10_gcm_encrypt) |
| |
| cmpdi 5, 0 |
| ble __Invalid_msg_len |
| |
| SAVE_REGS |
| LOAD_HASH_TABLE |
| |
| # initialize ICB: GHASH( IV ), IV - r7 |
| lxvb16x 30+32, 0, 7 # load IV - v30 |
| |
| mr 14, 3 |
| mr 9, 4 |
| |
| # counter 1 |
| vxor 31, 31, 31 |
| vspltisb 22, 1 |
| vsldoi 31, 31, 22,1 # counter 1 |
| |
| addis 11, 2, permx@toc@ha |
| addi 11, 11, permx@toc@l |
| lxv 10, 0(11) # vs10: vpermxor vector |
| li 11, 0 |
| |
| # load 9 round keys to VSR |
| lxv 0, 0(6) # round key 0 |
| lxv 1, 16(6) # round key 1 |
| lxv 2, 32(6) # round key 2 |
| lxv 3, 48(6) # round key 3 |
| lxv 4, 64(6) # round key 4 |
| lxv 5, 80(6) # round key 5 |
| lxv 6, 96(6) # round key 6 |
| lxv 7, 112(6) # round key 7 |
| lxv 8, 128(6) # round key 8 |
| |
| # load rounds - 10 (128), 12 (192), 14 (256) |
| lwz 23, 240(6) # n rounds |
| li 24, 1 # encrypt |
| |
| __Process_encrypt: |
| # |
| # Process different blocks |
| # |
| ld 12, 56(7) |
| cmpdi 12, 0 |
| bgt __Do_combine_enc |
| cmpdi 5, 128 |
| blt __Process_more_enc |
| |
| # |
| # Process 8x AES/GCM blocks |
| # |
| __Process_8x_enc: |
| # 8x blcoks |
| li 10, 128 |
| divdu 12, 5, 10 # n 128 bytes-blocks |
| |
| addi 12, 12, -1 # loop - 1 |
| |
| vmr 15, 30 # first state: IV |
| vadduwm 16, 15, 31 # state + counter |
| vadduwm 17, 16, 31 |
| vadduwm 18, 17, 31 |
| vadduwm 19, 18, 31 |
| vadduwm 20, 19, 31 |
| vadduwm 21, 20, 31 |
| vadduwm 22, 21, 31 |
| xxlor 9, 32+22, 32+22 # save last state |
| |
| # vxor state, state, w # addroundkey |
| xxlor 32+29, 0, 0 |
| vxor 15, 15, 29 # IV + round key - add round key 0 |
| vxor 16, 16, 29 |
| vxor 17, 17, 29 |
| vxor 18, 18, 29 |
| vxor 19, 19, 29 |
| vxor 20, 20, 29 |
| vxor 21, 21, 29 |
| vxor 22, 22, 29 |
| |
| li 15, 16 |
| li 16, 32 |
| li 17, 48 |
| li 18, 64 |
| li 19, 80 |
| li 20, 96 |
| li 21, 112 |
| |
| # |
| # Pre-compute first 8 AES state and leave 1/3/5 more rounds |
| # for the loop. |
| # |
| addi 22, 23, -9 # process 8 keys |
| mtctr 22 # AES key loop |
| addi 10, 6, 144 |
| |
| LOOP_8AES_STATE # process 8 AES keys |
| |
| __PreLoop_aes_state: |
| lxv 32+1, 0(10) # round key |
| AES_CIPHER_8x vcipher 15 1 |
| addi 10, 10, 16 |
| bdnz __PreLoop_aes_state |
| lxv 32+1, 0(10) # last round key (v1) |
| |
| cmpdi 12, 0 # Only one loop (8 block) |
| beq __Finish_ghash |
| |
| # |
| # Loop 8x blocks and compute ghash |
| # |
| __Loop_8x_block_enc: |
| vcipherlast 15, 15, 1 |
| vcipherlast 16, 16, 1 |
| vcipherlast 17, 17, 1 |
| vcipherlast 18, 18, 1 |
| vcipherlast 19, 19, 1 |
| vcipherlast 20, 20, 1 |
| vcipherlast 21, 21, 1 |
| vcipherlast 22, 22, 1 |
| |
| lxvb16x 32+23, 0, 14 # load block |
| lxvb16x 32+24, 15, 14 # load block |
| lxvb16x 32+25, 16, 14 # load block |
| lxvb16x 32+26, 17, 14 # load block |
| lxvb16x 32+27, 18, 14 # load block |
| lxvb16x 32+28, 19, 14 # load block |
| lxvb16x 32+29, 20, 14 # load block |
| lxvb16x 32+30, 21, 14 # load block |
| addi 14, 14, 128 |
| |
| vxor 15, 15, 23 |
| vxor 16, 16, 24 |
| vxor 17, 17, 25 |
| vxor 18, 18, 26 |
| vxor 19, 19, 27 |
| vxor 20, 20, 28 |
| vxor 21, 21, 29 |
| vxor 22, 22, 30 |
| |
| stxvb16x 47, 0, 9 # store output |
| stxvb16x 48, 15, 9 # store output |
| stxvb16x 49, 16, 9 # store output |
| stxvb16x 50, 17, 9 # store output |
| stxvb16x 51, 18, 9 # store output |
| stxvb16x 52, 19, 9 # store output |
| stxvb16x 53, 20, 9 # store output |
| stxvb16x 54, 21, 9 # store output |
| addi 9, 9, 128 |
| |
| # ghash here |
| vxor 15, 15, 0 |
| PPC_GHASH4x 0, 15, 16, 17, 18 |
| |
| vxor 19, 19, 0 |
| PPC_GHASH4x 0, 19, 20, 21, 22 |
| |
| xxlor 32+15, 9, 9 # last state |
| vadduwm 15, 15, 31 # state + counter |
| vadduwm 16, 15, 31 |
| vadduwm 17, 16, 31 |
| vadduwm 18, 17, 31 |
| vadduwm 19, 18, 31 |
| vadduwm 20, 19, 31 |
| vadduwm 21, 20, 31 |
| vadduwm 22, 21, 31 |
| xxlor 9, 32+22, 32+22 # save last state |
| |
| xxlor 32+27, 0, 0 # restore roundkey 0 |
| vxor 15, 15, 27 # IV + round key - add round key 0 |
| vxor 16, 16, 27 |
| vxor 17, 17, 27 |
| vxor 18, 18, 27 |
| vxor 19, 19, 27 |
| vxor 20, 20, 27 |
| vxor 21, 21, 27 |
| vxor 22, 22, 27 |
| |
| addi 5, 5, -128 |
| addi 11, 11, 128 |
| |
| LOOP_8AES_STATE # process 8 AES keys |
| mtctr 22 # AES key loop |
| addi 10, 6, 144 |
| __LastLoop_aes_state: |
| lxv 32+1, 0(10) # round key |
| AES_CIPHER_8x vcipher 15 1 |
| addi 10, 10, 16 |
| bdnz __LastLoop_aes_state |
| lxv 32+1, 0(10) # last round key (v1) |
| |
| addi 12, 12, -1 |
| cmpdi 12, 0 |
| bne __Loop_8x_block_enc |
| |
| __Finish_ghash: |
| vcipherlast 15, 15, 1 |
| vcipherlast 16, 16, 1 |
| vcipherlast 17, 17, 1 |
| vcipherlast 18, 18, 1 |
| vcipherlast 19, 19, 1 |
| vcipherlast 20, 20, 1 |
| vcipherlast 21, 21, 1 |
| vcipherlast 22, 22, 1 |
| |
| lxvb16x 32+23, 0, 14 # load block |
| lxvb16x 32+24, 15, 14 # load block |
| lxvb16x 32+25, 16, 14 # load block |
| lxvb16x 32+26, 17, 14 # load block |
| lxvb16x 32+27, 18, 14 # load block |
| lxvb16x 32+28, 19, 14 # load block |
| lxvb16x 32+29, 20, 14 # load block |
| lxvb16x 32+30, 21, 14 # load block |
| addi 14, 14, 128 |
| |
| vxor 15, 15, 23 |
| vxor 16, 16, 24 |
| vxor 17, 17, 25 |
| vxor 18, 18, 26 |
| vxor 19, 19, 27 |
| vxor 20, 20, 28 |
| vxor 21, 21, 29 |
| vxor 22, 22, 30 |
| |
| stxvb16x 47, 0, 9 # store output |
| stxvb16x 48, 15, 9 # store output |
| stxvb16x 49, 16, 9 # store output |
| stxvb16x 50, 17, 9 # store output |
| stxvb16x 51, 18, 9 # store output |
| stxvb16x 52, 19, 9 # store output |
| stxvb16x 53, 20, 9 # store output |
| stxvb16x 54, 21, 9 # store output |
| addi 9, 9, 128 |
| |
| vxor 15, 15, 0 |
| PPC_GHASH4x 0, 15, 16, 17, 18 |
| |
| vxor 19, 19, 0 |
| PPC_GHASH4x 0, 19, 20, 21, 22 |
| |
| xxlor 30+32, 9, 9 # last ctr |
| vadduwm 30, 30, 31 # increase ctr |
| stxvb16x 32+30, 0, 7 # update IV |
| stxvb16x 32+0, 0, 8 # update Xi |
| |
| addi 5, 5, -128 |
| addi 11, 11, 128 |
| |
| # |
| # Done 8x blocks |
| # |
| |
| cmpdi 5, 0 |
| beq aes_gcm_out |
| |
| __Process_more_enc: |
| li 24, 1 # encrypt |
| bl aes_gcm_crypt_1x |
| cmpdi 5, 0 |
| beq aes_gcm_out |
| |
| bl __Process_partial |
| cmpdi 5, 0 |
| beq aes_gcm_out |
| __Do_combine_enc: |
| bl __Combine_partial |
| cmpdi 5, 0 |
| bgt __Process_encrypt |
| b aes_gcm_out |
| |
| SYM_FUNC_END(aes_p10_gcm_encrypt) |
| |
| ################################################################################ |
| # aes_p10_gcm_decrypt (const void *inp, void *out, size_t len, |
| # const char *rk, unsigned char iv[16], void *Xip); |
| # 8x Decrypt |
| # |
| ################################################################################ |
| SYM_FUNC_START(aes_p10_gcm_decrypt) |
| |
| cmpdi 5, 0 |
| ble __Invalid_msg_len |
| |
| SAVE_REGS |
| LOAD_HASH_TABLE |
| |
| # initialize ICB: GHASH( IV ), IV - r7 |
| lxvb16x 30+32, 0, 7 # load IV - v30 |
| |
| mr 14, 3 |
| mr 9, 4 |
| |
| # counter 1 |
| vxor 31, 31, 31 |
| vspltisb 22, 1 |
| vsldoi 31, 31, 22,1 # counter 1 |
| |
| addis 11, 2, permx@toc@ha |
| addi 11, 11, permx@toc@l |
| lxv 10, 0(11) # vs10: vpermxor vector |
| li 11, 0 |
| |
| # load 9 round keys to VSR |
| lxv 0, 0(6) # round key 0 |
| lxv 1, 16(6) # round key 1 |
| lxv 2, 32(6) # round key 2 |
| lxv 3, 48(6) # round key 3 |
| lxv 4, 64(6) # round key 4 |
| lxv 5, 80(6) # round key 5 |
| lxv 6, 96(6) # round key 6 |
| lxv 7, 112(6) # round key 7 |
| lxv 8, 128(6) # round key 8 |
| |
| # load rounds - 10 (128), 12 (192), 14 (256) |
| lwz 23, 240(6) # n rounds |
| li 24, 0 # decrypt |
| |
| __Process_decrypt: |
| # |
| # Process different blocks |
| # |
| ld 12, 56(7) |
| cmpdi 12, 0 |
| bgt __Do_combine_dec |
| cmpdi 5, 128 |
| blt __Process_more_dec |
| |
| # |
| # Process 8x AES/GCM blocks |
| # |
| __Process_8x_dec: |
| # 8x blcoks |
| li 10, 128 |
| divdu 12, 5, 10 # n 128 bytes-blocks |
| |
| addi 12, 12, -1 # loop - 1 |
| |
| vmr 15, 30 # first state: IV |
| vadduwm 16, 15, 31 # state + counter |
| vadduwm 17, 16, 31 |
| vadduwm 18, 17, 31 |
| vadduwm 19, 18, 31 |
| vadduwm 20, 19, 31 |
| vadduwm 21, 20, 31 |
| vadduwm 22, 21, 31 |
| xxlor 9, 32+22, 32+22 # save last state |
| |
| # vxor state, state, w # addroundkey |
| xxlor 32+29, 0, 0 |
| vxor 15, 15, 29 # IV + round key - add round key 0 |
| vxor 16, 16, 29 |
| vxor 17, 17, 29 |
| vxor 18, 18, 29 |
| vxor 19, 19, 29 |
| vxor 20, 20, 29 |
| vxor 21, 21, 29 |
| vxor 22, 22, 29 |
| |
| li 15, 16 |
| li 16, 32 |
| li 17, 48 |
| li 18, 64 |
| li 19, 80 |
| li 20, 96 |
| li 21, 112 |
| |
| # |
| # Pre-compute first 8 AES state and leave 1/3/5 more rounds |
| # for the loop. |
| # |
| addi 22, 23, -9 # process 8 keys |
| mtctr 22 # AES key loop |
| addi 10, 6, 144 |
| |
| LOOP_8AES_STATE # process 8 AES keys |
| |
| __PreLoop_aes_state_dec: |
| lxv 32+1, 0(10) # round key |
| AES_CIPHER_8x vcipher 15 1 |
| addi 10, 10, 16 |
| bdnz __PreLoop_aes_state_dec |
| lxv 32+1, 0(10) # last round key (v1) |
| |
| cmpdi 12, 0 # Only one loop (8 block) |
| beq __Finish_ghash_dec |
| |
| # |
| # Loop 8x blocks and compute ghash |
| # |
| __Loop_8x_block_dec: |
| vcipherlast 15, 15, 1 |
| vcipherlast 16, 16, 1 |
| vcipherlast 17, 17, 1 |
| vcipherlast 18, 18, 1 |
| vcipherlast 19, 19, 1 |
| vcipherlast 20, 20, 1 |
| vcipherlast 21, 21, 1 |
| vcipherlast 22, 22, 1 |
| |
| lxvb16x 32+23, 0, 14 # load block |
| lxvb16x 32+24, 15, 14 # load block |
| lxvb16x 32+25, 16, 14 # load block |
| lxvb16x 32+26, 17, 14 # load block |
| lxvb16x 32+27, 18, 14 # load block |
| lxvb16x 32+28, 19, 14 # load block |
| lxvb16x 32+29, 20, 14 # load block |
| lxvb16x 32+30, 21, 14 # load block |
| addi 14, 14, 128 |
| |
| vxor 15, 15, 23 |
| vxor 16, 16, 24 |
| vxor 17, 17, 25 |
| vxor 18, 18, 26 |
| vxor 19, 19, 27 |
| vxor 20, 20, 28 |
| vxor 21, 21, 29 |
| vxor 22, 22, 30 |
| |
| stxvb16x 47, 0, 9 # store output |
| stxvb16x 48, 15, 9 # store output |
| stxvb16x 49, 16, 9 # store output |
| stxvb16x 50, 17, 9 # store output |
| stxvb16x 51, 18, 9 # store output |
| stxvb16x 52, 19, 9 # store output |
| stxvb16x 53, 20, 9 # store output |
| stxvb16x 54, 21, 9 # store output |
| |
| addi 9, 9, 128 |
| |
| vmr 15, 23 |
| vmr 16, 24 |
| vmr 17, 25 |
| vmr 18, 26 |
| vmr 19, 27 |
| vmr 20, 28 |
| vmr 21, 29 |
| vmr 22, 30 |
| |
| # ghash here |
| vxor 15, 15, 0 |
| PPC_GHASH4x 0, 15, 16, 17, 18 |
| |
| vxor 19, 19, 0 |
| PPC_GHASH4x 0, 19, 20, 21, 22 |
| |
| xxlor 32+15, 9, 9 # last state |
| vadduwm 15, 15, 31 # state + counter |
| vadduwm 16, 15, 31 |
| vadduwm 17, 16, 31 |
| vadduwm 18, 17, 31 |
| vadduwm 19, 18, 31 |
| vadduwm 20, 19, 31 |
| vadduwm 21, 20, 31 |
| vadduwm 22, 21, 31 |
| xxlor 9, 32+22, 32+22 # save last state |
| |
| xxlor 32+27, 0, 0 # restore roundkey 0 |
| vxor 15, 15, 27 # IV + round key - add round key 0 |
| vxor 16, 16, 27 |
| vxor 17, 17, 27 |
| vxor 18, 18, 27 |
| vxor 19, 19, 27 |
| vxor 20, 20, 27 |
| vxor 21, 21, 27 |
| vxor 22, 22, 27 |
| |
| addi 5, 5, -128 |
| addi 11, 11, 128 |
| |
| LOOP_8AES_STATE # process 8 AES keys |
| mtctr 22 # AES key loop |
| addi 10, 6, 144 |
| __LastLoop_aes_state_dec: |
| lxv 32+1, 0(10) # round key |
| AES_CIPHER_8x vcipher 15 1 |
| addi 10, 10, 16 |
| bdnz __LastLoop_aes_state_dec |
| lxv 32+1, 0(10) # last round key (v1) |
| |
| addi 12, 12, -1 |
| cmpdi 12, 0 |
| bne __Loop_8x_block_dec |
| |
| __Finish_ghash_dec: |
| vcipherlast 15, 15, 1 |
| vcipherlast 16, 16, 1 |
| vcipherlast 17, 17, 1 |
| vcipherlast 18, 18, 1 |
| vcipherlast 19, 19, 1 |
| vcipherlast 20, 20, 1 |
| vcipherlast 21, 21, 1 |
| vcipherlast 22, 22, 1 |
| |
| lxvb16x 32+23, 0, 14 # load block |
| lxvb16x 32+24, 15, 14 # load block |
| lxvb16x 32+25, 16, 14 # load block |
| lxvb16x 32+26, 17, 14 # load block |
| lxvb16x 32+27, 18, 14 # load block |
| lxvb16x 32+28, 19, 14 # load block |
| lxvb16x 32+29, 20, 14 # load block |
| lxvb16x 32+30, 21, 14 # load block |
| addi 14, 14, 128 |
| |
| vxor 15, 15, 23 |
| vxor 16, 16, 24 |
| vxor 17, 17, 25 |
| vxor 18, 18, 26 |
| vxor 19, 19, 27 |
| vxor 20, 20, 28 |
| vxor 21, 21, 29 |
| vxor 22, 22, 30 |
| |
| stxvb16x 47, 0, 9 # store output |
| stxvb16x 48, 15, 9 # store output |
| stxvb16x 49, 16, 9 # store output |
| stxvb16x 50, 17, 9 # store output |
| stxvb16x 51, 18, 9 # store output |
| stxvb16x 52, 19, 9 # store output |
| stxvb16x 53, 20, 9 # store output |
| stxvb16x 54, 21, 9 # store output |
| addi 9, 9, 128 |
| |
| #vmr 15, 23 |
| vxor 15, 23, 0 |
| vmr 16, 24 |
| vmr 17, 25 |
| vmr 18, 26 |
| vmr 19, 27 |
| vmr 20, 28 |
| vmr 21, 29 |
| vmr 22, 30 |
| |
| #vxor 15, 15, 0 |
| PPC_GHASH4x 0, 15, 16, 17, 18 |
| |
| vxor 19, 19, 0 |
| PPC_GHASH4x 0, 19, 20, 21, 22 |
| |
| xxlor 30+32, 9, 9 # last ctr |
| vadduwm 30, 30, 31 # increase ctr |
| stxvb16x 32+30, 0, 7 # update IV |
| stxvb16x 32+0, 0, 8 # update Xi |
| |
| addi 5, 5, -128 |
| addi 11, 11, 128 |
| |
| # |
| # Done 8x blocks |
| # |
| |
| cmpdi 5, 0 |
| beq aes_gcm_out |
| |
| __Process_more_dec: |
| li 24, 0 # decrypt |
| bl aes_gcm_crypt_1x |
| cmpdi 5, 0 |
| beq aes_gcm_out |
| |
| bl __Process_partial |
| cmpdi 5, 0 |
| beq aes_gcm_out |
| __Do_combine_dec: |
| bl __Combine_partial |
| cmpdi 5, 0 |
| bgt __Process_decrypt |
| b aes_gcm_out |
| SYM_FUNC_END(aes_p10_gcm_decrypt) |
| |
| SYM_FUNC_START_LOCAL(aes_gcm_out) |
| |
| mr 3, 11 # return count |
| |
| RESTORE_REGS |
| blr |
| |
| __Invalid_msg_len: |
| li 3, 0 |
| blr |
| SYM_FUNC_END(aes_gcm_out) |
| |
| SYM_DATA_START_LOCAL(PERMX) |
| .align 4 |
| # for vector permute and xor |
| permx: |
| .long 0x4c5d6e7f, 0x08192a3b, 0xc4d5e6f7, 0x8091a2b3 |
| SYM_DATA_END(permx) |
