Thomas Gleixner | d2912cb | 2019-06-04 10:11:33 +0200 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 2 | /* |
| 3 | * aesce-ccm-core.S - AES-CCM transform for ARMv8 with Crypto Extensions |
| 4 | * |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 5 | * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 6 | */ |
| 7 | |
| 8 | #include <linux/linkage.h> |
Ard Biesheuvel | 56e4e76 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 9 | #include <asm/assembler.h> |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 10 | |
| 11 | .text |
| 12 | .arch armv8-a+crypto |
| 13 | |
| 14 | /* |
| 15 | * void ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes, |
| 16 | * u32 *macp, u8 const rk[], u32 rounds); |
| 17 | */ |
Mark Brown | 0e89640 | 2019-12-13 15:49:10 +0000 | [diff] [blame] | 18 | SYM_FUNC_START(ce_aes_ccm_auth_data) |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 19 | ldr w8, [x3] /* leftover from prev round? */ |
Ard Biesheuvel | 56e4e76 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 20 | ld1 {v0.16b}, [x0] /* load mac */ |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 21 | cbz w8, 1f |
| 22 | sub w8, w8, #16 |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 23 | eor v1.16b, v1.16b, v1.16b |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 24 | 0: ldrb w7, [x1], #1 /* get 1 byte of input */ |
| 25 | subs w2, w2, #1 |
| 26 | add w8, w8, #1 |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 27 | ins v1.b[0], w7 |
| 28 | ext v1.16b, v1.16b, v1.16b, #1 /* rotate in the input bytes */ |
| 29 | beq 8f /* out of input? */ |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 30 | cbnz w8, 0b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 31 | eor v0.16b, v0.16b, v1.16b |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 32 | 1: ld1 {v3.4s}, [x4] /* load first round key */ |
| 33 | prfm pldl1strm, [x1] |
| 34 | cmp w5, #12 /* which key size? */ |
| 35 | add x6, x4, #16 |
| 36 | sub w7, w5, #2 /* modified # of rounds */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 37 | bmi 2f |
| 38 | bne 5f |
| 39 | mov v5.16b, v3.16b |
| 40 | b 4f |
| 41 | 2: mov v4.16b, v3.16b |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 42 | ld1 {v5.4s}, [x6], #16 /* load 2nd round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 43 | 3: aese v0.16b, v4.16b |
| 44 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 45 | 4: ld1 {v3.4s}, [x6], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 46 | aese v0.16b, v5.16b |
| 47 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 48 | 5: ld1 {v4.4s}, [x6], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 49 | subs w7, w7, #3 |
| 50 | aese v0.16b, v3.16b |
| 51 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 52 | ld1 {v5.4s}, [x6], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 53 | bpl 3b |
| 54 | aese v0.16b, v4.16b |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 55 | subs w2, w2, #16 /* last data? */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 56 | eor v0.16b, v0.16b, v5.16b /* final round */ |
| 57 | bmi 6f |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 58 | ld1 {v1.16b}, [x1], #16 /* load next input block */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 59 | eor v0.16b, v0.16b, v1.16b /* xor with mac */ |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 60 | bne 1b |
| 61 | 6: st1 {v0.16b}, [x0] /* store mac */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 62 | beq 10f |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 63 | adds w2, w2, #16 |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 64 | beq 10f |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 65 | mov w8, w2 |
| 66 | 7: ldrb w7, [x1], #1 |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 67 | umov w6, v0.b[0] |
| 68 | eor w6, w6, w7 |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 69 | strb w6, [x0], #1 |
| 70 | subs w2, w2, #1 |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 71 | beq 10f |
| 72 | ext v0.16b, v0.16b, v0.16b, #1 /* rotate out the mac bytes */ |
| 73 | b 7b |
Ard Biesheuvel | eaf46ed | 2019-01-24 17:33:45 +0100 | [diff] [blame] | 74 | 8: cbz w8, 91f |
| 75 | mov w7, w8 |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 76 | add w8, w8, #16 |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 77 | 9: ext v1.16b, v1.16b, v1.16b, #1 |
| 78 | adds w7, w7, #1 |
| 79 | bne 9b |
Ard Biesheuvel | eaf46ed | 2019-01-24 17:33:45 +0100 | [diff] [blame] | 80 | 91: eor v0.16b, v0.16b, v1.16b |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 81 | st1 {v0.16b}, [x0] |
| 82 | 10: str w8, [x3] |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 83 | ret |
Mark Brown | 0e89640 | 2019-12-13 15:49:10 +0000 | [diff] [blame] | 84 | SYM_FUNC_END(ce_aes_ccm_auth_data) |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 85 | |
| 86 | /* |
| 87 | * void ce_aes_ccm_final(u8 mac[], u8 const ctr[], u8 const rk[], |
| 88 | * u32 rounds); |
| 89 | */ |
Mark Brown | 0e89640 | 2019-12-13 15:49:10 +0000 | [diff] [blame] | 90 | SYM_FUNC_START(ce_aes_ccm_final) |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 91 | ld1 {v3.4s}, [x2], #16 /* load first round key */ |
Ard Biesheuvel | 56e4e76 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 92 | ld1 {v0.16b}, [x0] /* load mac */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 93 | cmp w3, #12 /* which key size? */ |
| 94 | sub w3, w3, #2 /* modified # of rounds */ |
Ard Biesheuvel | 56e4e76 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 95 | ld1 {v1.16b}, [x1] /* load 1st ctriv */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 96 | bmi 0f |
| 97 | bne 3f |
| 98 | mov v5.16b, v3.16b |
| 99 | b 2f |
| 100 | 0: mov v4.16b, v3.16b |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 101 | 1: ld1 {v5.4s}, [x2], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 102 | aese v0.16b, v4.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 103 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | 4a97abd4 | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 104 | aese v1.16b, v4.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 105 | aesmc v1.16b, v1.16b |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 106 | 2: ld1 {v3.4s}, [x2], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 107 | aese v0.16b, v5.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 108 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | 4a97abd4 | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 109 | aese v1.16b, v5.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 110 | aesmc v1.16b, v1.16b |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 111 | 3: ld1 {v4.4s}, [x2], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 112 | subs w3, w3, #3 |
| 113 | aese v0.16b, v3.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 114 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | 4a97abd4 | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 115 | aese v1.16b, v3.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 116 | aesmc v1.16b, v1.16b |
| 117 | bpl 1b |
| 118 | aese v0.16b, v4.16b |
| 119 | aese v1.16b, v4.16b |
| 120 | /* final round key cancels out */ |
| 121 | eor v0.16b, v0.16b, v1.16b /* en-/decrypt the mac */ |
Ard Biesheuvel | 56e4e76 | 2016-10-11 19:15:17 +0100 | [diff] [blame] | 122 | st1 {v0.16b}, [x0] /* store result */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 123 | ret |
Mark Brown | 0e89640 | 2019-12-13 15:49:10 +0000 | [diff] [blame] | 124 | SYM_FUNC_END(ce_aes_ccm_final) |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 125 | |
| 126 | .macro aes_ccm_do_crypt,enc |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 127 | ldr x8, [x6, #8] /* load lower ctr */ |
| 128 | ld1 {v0.16b}, [x5] /* load mac */ |
| 129 | CPU_LE( rev x8, x8 ) /* keep swabbed ctr in reg */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 130 | 0: /* outer loop */ |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 131 | ld1 {v1.8b}, [x6] /* load upper ctr */ |
| 132 | prfm pldl1strm, [x1] |
| 133 | add x8, x8, #1 |
| 134 | rev x9, x8 |
| 135 | cmp w4, #12 /* which key size? */ |
| 136 | sub w7, w4, #2 /* get modified # of rounds */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 137 | ins v1.d[1], x9 /* no carry in lower ctr */ |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 138 | ld1 {v3.4s}, [x3] /* load first round key */ |
| 139 | add x10, x3, #16 |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 140 | bmi 1f |
| 141 | bne 4f |
| 142 | mov v5.16b, v3.16b |
| 143 | b 3f |
| 144 | 1: mov v4.16b, v3.16b |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 145 | ld1 {v5.4s}, [x10], #16 /* load 2nd round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 146 | 2: /* inner loop: 3 rounds, 2x interleaved */ |
| 147 | aese v0.16b, v4.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 148 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | 4a97abd4 | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 149 | aese v1.16b, v4.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 150 | aesmc v1.16b, v1.16b |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 151 | 3: ld1 {v3.4s}, [x10], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 152 | aese v0.16b, v5.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 153 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | 4a97abd4 | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 154 | aese v1.16b, v5.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 155 | aesmc v1.16b, v1.16b |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 156 | 4: ld1 {v4.4s}, [x10], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 157 | subs w7, w7, #3 |
| 158 | aese v0.16b, v3.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 159 | aesmc v0.16b, v0.16b |
Ard Biesheuvel | 4a97abd4 | 2015-03-17 18:05:13 +0000 | [diff] [blame] | 160 | aese v1.16b, v3.16b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 161 | aesmc v1.16b, v1.16b |
Ard Biesheuvel | f402e311 | 2017-07-24 11:28:10 +0100 | [diff] [blame] | 162 | ld1 {v5.4s}, [x10], #16 /* load next round key */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 163 | bpl 2b |
| 164 | aese v0.16b, v4.16b |
| 165 | aese v1.16b, v4.16b |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 166 | subs w2, w2, #16 |
| 167 | bmi 6f /* partial block? */ |
| 168 | ld1 {v2.16b}, [x1], #16 /* load next input block */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 169 | .if \enc == 1 |
| 170 | eor v2.16b, v2.16b, v5.16b /* final round enc+mac */ |
| 171 | eor v1.16b, v1.16b, v2.16b /* xor with crypted ctr */ |
| 172 | .else |
| 173 | eor v2.16b, v2.16b, v1.16b /* xor with crypted ctr */ |
| 174 | eor v1.16b, v2.16b, v5.16b /* final round enc */ |
| 175 | .endif |
| 176 | eor v0.16b, v0.16b, v2.16b /* xor mac with pt ^ rk[last] */ |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 177 | st1 {v1.16b}, [x0], #16 /* write output block */ |
| 178 | bne 0b |
| 179 | CPU_LE( rev x8, x8 ) |
| 180 | st1 {v0.16b}, [x5] /* store mac */ |
| 181 | str x8, [x6, #8] /* store lsb end of ctr (BE) */ |
| 182 | 5: ret |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 183 | |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 184 | 6: eor v0.16b, v0.16b, v5.16b /* final round mac */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 185 | eor v1.16b, v1.16b, v5.16b /* final round enc */ |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 186 | st1 {v0.16b}, [x5] /* store mac */ |
| 187 | add w2, w2, #16 /* process partial tail block */ |
| 188 | 7: ldrb w9, [x1], #1 /* get 1 byte of input */ |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 189 | umov w6, v1.b[0] /* get top crypted ctr byte */ |
| 190 | umov w7, v0.b[0] /* get top mac byte */ |
| 191 | .if \enc == 1 |
| 192 | eor w7, w7, w9 |
| 193 | eor w9, w9, w6 |
| 194 | .else |
| 195 | eor w9, w9, w6 |
| 196 | eor w7, w7, w9 |
| 197 | .endif |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 198 | strb w9, [x0], #1 /* store out byte */ |
| 199 | strb w7, [x5], #1 /* store mac byte */ |
| 200 | subs w2, w2, #1 |
| 201 | beq 5b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 202 | ext v0.16b, v0.16b, v0.16b, #1 /* shift out mac byte */ |
| 203 | ext v1.16b, v1.16b, v1.16b, #1 /* shift out ctr byte */ |
Ard Biesheuvel | f10dc56 | 2018-07-29 16:52:30 +0200 | [diff] [blame] | 204 | b 7b |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 205 | .endm |
| 206 | |
| 207 | /* |
| 208 | * void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes, |
| 209 | * u8 const rk[], u32 rounds, u8 mac[], |
| 210 | * u8 ctr[]); |
| 211 | * void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes, |
| 212 | * u8 const rk[], u32 rounds, u8 mac[], |
| 213 | * u8 ctr[]); |
| 214 | */ |
Mark Brown | 0e89640 | 2019-12-13 15:49:10 +0000 | [diff] [blame] | 215 | SYM_FUNC_START(ce_aes_ccm_encrypt) |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 216 | aes_ccm_do_crypt 1 |
Mark Brown | 0e89640 | 2019-12-13 15:49:10 +0000 | [diff] [blame] | 217 | SYM_FUNC_END(ce_aes_ccm_encrypt) |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 218 | |
Mark Brown | 0e89640 | 2019-12-13 15:49:10 +0000 | [diff] [blame] | 219 | SYM_FUNC_START(ce_aes_ccm_decrypt) |
Ard Biesheuvel | a3fd821 | 2014-02-10 11:26:29 +0100 | [diff] [blame] | 220 | aes_ccm_do_crypt 0 |
Mark Brown | 0e89640 | 2019-12-13 15:49:10 +0000 | [diff] [blame] | 221 | SYM_FUNC_END(ce_aes_ccm_decrypt) |