Ard Biesheuvel | 49788fe | 2014-03-21 10:19:17 +0100 | [diff] [blame] | 1 | /* |
| 2 | * linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with |
| 3 | * Crypto Extensions |
| 4 | * |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 5 | * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> |
Ard Biesheuvel | 49788fe | 2014-03-21 10:19:17 +0100 | [diff] [blame] | 6 | * |
| 7 | * This program is free software; you can redistribute it and/or modify |
| 8 | * it under the terms of the GNU General Public License version 2 as |
| 9 | * published by the Free Software Foundation. |
| 10 | */ |
| 11 | |
| 12 | #include <linux/linkage.h> |
Ard Biesheuvel | caf4b9e | 2016-10-11 19:15:19 +0100 | [diff] [blame] | 13 | #include <asm/assembler.h> |
Ard Biesheuvel | 49788fe | 2014-03-21 10:19:17 +0100 | [diff] [blame] | 14 | |
| 15 | #define AES_ENTRY(func) ENTRY(ce_ ## func) |
| 16 | #define AES_ENDPROC(func) ENDPROC(ce_ ## func) |
| 17 | |
| 18 | .arch armv8-a+crypto |
| 19 | |
Ard Biesheuvel | 2e5d2f3 | 2018-09-10 16:41:15 +0200 | [diff] [blame] | 20 | xtsmask .req v16 |
| 21 | |
| 22 | .macro xts_reload_mask, tmp |
| 23 | .endm |
| 24 | |
Ard Biesheuvel | 49788fe | 2014-03-21 10:19:17 +0100 | [diff] [blame] | 25 | /* preload all round keys */ |
| 26 | .macro load_round_keys, rounds, rk |
| 27 | cmp \rounds, #12 |
| 28 | blo 2222f /* 128 bits */ |
| 29 | beq 1111f /* 192 bits */ |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 30 | ld1 {v17.4s-v18.4s}, [\rk], #32 |
| 31 | 1111: ld1 {v19.4s-v20.4s}, [\rk], #32 |
| 32 | 2222: ld1 {v21.4s-v24.4s}, [\rk], #64 |
| 33 | ld1 {v25.4s-v28.4s}, [\rk], #64 |
| 34 | ld1 {v29.4s-v31.4s}, [\rk] |
Ard Biesheuvel | 49788fe | 2014-03-21 10:19:17 +0100 | [diff] [blame] | 35 | .endm |
| 36 | |
| 37 | /* prepare for encryption with key in rk[] */ |
Ard Biesheuvel | 0c8f838 | 2018-04-30 18:18:24 +0200 | [diff] [blame] | 38 | .macro enc_prepare, rounds, rk, temp |
| 39 | mov \temp, \rk |
| 40 | load_round_keys \rounds, \temp |
Ard Biesheuvel | 49788fe | 2014-03-21 10:19:17 +0100 | [diff] [blame] | 41 | .endm |
| 42 | |
| 43 | /* prepare for encryption (again) but with new key in rk[] */ |
Ard Biesheuvel | 0c8f838 | 2018-04-30 18:18:24 +0200 | [diff] [blame] | 44 | .macro enc_switch_key, rounds, rk, temp |
| 45 | mov \temp, \rk |
| 46 | load_round_keys \rounds, \temp |
Ard Biesheuvel | 49788fe | 2014-03-21 10:19:17 +0100 | [diff] [blame] | 47 | .endm |
| 48 | |
| 49 | /* prepare for decryption with key in rk[] */ |
Ard Biesheuvel | 0c8f838 | 2018-04-30 18:18:24 +0200 | [diff] [blame] | 50 | .macro dec_prepare, rounds, rk, temp |
| 51 | mov \temp, \rk |
| 52 | load_round_keys \rounds, \temp |
Ard Biesheuvel | 49788fe | 2014-03-21 10:19:17 +0100 | [diff] [blame] | 53 | .endm |
| 54 | |
| 55 | .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3 |
| 56 | aes\de \i0\().16b, \k\().16b |
Ard Biesheuvel | 49788fe | 2014-03-21 10:19:17 +0100 | [diff] [blame] | 57 | aes\mc \i0\().16b, \i0\().16b |
| 58 | .ifnb \i1 |
Ard Biesheuvel | 4a97abd4 | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 59 | aes\de \i1\().16b, \k\().16b |
Ard Biesheuvel | 49788fe | 2014-03-21 10:19:17 +0100 | [diff] [blame] | 60 | aes\mc \i1\().16b, \i1\().16b |
| 61 | .ifnb \i3 |
Ard Biesheuvel | 4a97abd4 | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 62 | aes\de \i2\().16b, \k\().16b |
Ard Biesheuvel | 49788fe | 2014-03-21 10:19:17 +0100 | [diff] [blame] | 63 | aes\mc \i2\().16b, \i2\().16b |
Ard Biesheuvel | 4a97abd4 | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 64 | aes\de \i3\().16b, \k\().16b |
Ard Biesheuvel | 49788fe | 2014-03-21 10:19:17 +0100 | [diff] [blame] | 65 | aes\mc \i3\().16b, \i3\().16b |
| 66 | .endif |
| 67 | .endif |
| 68 | .endm |
| 69 | |
| 70 | /* up to 4 interleaved encryption rounds with the same round key */ |
| 71 | .macro round_Nx, enc, k, i0, i1, i2, i3 |
| 72 | .ifc \enc, e |
| 73 | do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3 |
| 74 | .else |
| 75 | do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3 |
| 76 | .endif |
| 77 | .endm |
| 78 | |
| 79 | /* up to 4 interleaved final rounds */ |
| 80 | .macro fin_round_Nx, de, k, k2, i0, i1, i2, i3 |
| 81 | aes\de \i0\().16b, \k\().16b |
| 82 | .ifnb \i1 |
| 83 | aes\de \i1\().16b, \k\().16b |
| 84 | .ifnb \i3 |
| 85 | aes\de \i2\().16b, \k\().16b |
| 86 | aes\de \i3\().16b, \k\().16b |
| 87 | .endif |
| 88 | .endif |
| 89 | eor \i0\().16b, \i0\().16b, \k2\().16b |
| 90 | .ifnb \i1 |
| 91 | eor \i1\().16b, \i1\().16b, \k2\().16b |
| 92 | .ifnb \i3 |
| 93 | eor \i2\().16b, \i2\().16b, \k2\().16b |
| 94 | eor \i3\().16b, \i3\().16b, \k2\().16b |
| 95 | .endif |
| 96 | .endif |
| 97 | .endm |
| 98 | |
| 99 | /* up to 4 interleaved blocks */ |
| 100 | .macro do_block_Nx, enc, rounds, i0, i1, i2, i3 |
| 101 | cmp \rounds, #12 |
| 102 | blo 2222f /* 128 bits */ |
| 103 | beq 1111f /* 192 bits */ |
| 104 | round_Nx \enc, v17, \i0, \i1, \i2, \i3 |
| 105 | round_Nx \enc, v18, \i0, \i1, \i2, \i3 |
| 106 | 1111: round_Nx \enc, v19, \i0, \i1, \i2, \i3 |
| 107 | round_Nx \enc, v20, \i0, \i1, \i2, \i3 |
| 108 | 2222: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29 |
| 109 | round_Nx \enc, \key, \i0, \i1, \i2, \i3 |
| 110 | .endr |
| 111 | fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3 |
| 112 | .endm |
| 113 | |
| 114 | .macro encrypt_block, in, rounds, t0, t1, t2 |
| 115 | do_block_Nx e, \rounds, \in |
| 116 | .endm |
| 117 | |
| 118 | .macro encrypt_block2x, i0, i1, rounds, t0, t1, t2 |
| 119 | do_block_Nx e, \rounds, \i0, \i1 |
| 120 | .endm |
| 121 | |
| 122 | .macro encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 |
| 123 | do_block_Nx e, \rounds, \i0, \i1, \i2, \i3 |
| 124 | .endm |
| 125 | |
| 126 | .macro decrypt_block, in, rounds, t0, t1, t2 |
| 127 | do_block_Nx d, \rounds, \in |
| 128 | .endm |
| 129 | |
| 130 | .macro decrypt_block2x, i0, i1, rounds, t0, t1, t2 |
| 131 | do_block_Nx d, \rounds, \i0, \i1 |
| 132 | .endm |
| 133 | |
| 134 | .macro decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 |
| 135 | do_block_Nx d, \rounds, \i0, \i1, \i2, \i3 |
| 136 | .endm |
| 137 | |
| 138 | #include "aes-modes.S" |