blob: 384ccb00f9e15ab274d41900aab2e24dd0162117 [file] [log] [blame]
Thomas Gleixner1a59d1b82019-05-27 08:55:05 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +02002/*
Ingo Molnara97673a2018-12-03 10:47:34 +01003 * Glue Code for the AVX assembler implementation of the Cast5 Cipher
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +02004 *
5 * Copyright (C) 2012 Johannes Goetzfried
6 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +02007 */
8
Eric Biggers1e631832018-02-19 23:48:13 -08009#include <asm/crypto/glue_helper.h>
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020010#include <crypto/algapi.h>
11#include <crypto/cast5.h>
Eric Biggers1e631832018-02-19 23:48:13 -080012#include <crypto/internal/simd.h>
13#include <linux/crypto.h>
14#include <linux/err.h>
15#include <linux/module.h>
16#include <linux/types.h>
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020017
18#define CAST5_PARALLEL_BLOCKS 16
19
Jussi Kivilinnac12ab202012-10-20 15:06:56 +030020asmlinkage void cast5_ecb_enc_16way(struct cast5_ctx *ctx, u8 *dst,
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020021 const u8 *src);
Jussi Kivilinnac12ab202012-10-20 15:06:56 +030022asmlinkage void cast5_ecb_dec_16way(struct cast5_ctx *ctx, u8 *dst,
23 const u8 *src);
24asmlinkage void cast5_cbc_dec_16way(struct cast5_ctx *ctx, u8 *dst,
25 const u8 *src);
26asmlinkage void cast5_ctr_16way(struct cast5_ctx *ctx, u8 *dst, const u8 *src,
27 __be64 *iv);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020028
Eric Biggers1e631832018-02-19 23:48:13 -080029static int cast5_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key,
30 unsigned int keylen)
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020031{
Eric Biggers1e631832018-02-19 23:48:13 -080032 return cast5_setkey(&tfm->base, key, keylen);
33}
34
35static inline bool cast5_fpu_begin(bool fpu_enabled, struct skcipher_walk *walk,
36 unsigned int nbytes)
37{
Eric Biggers75d8a552018-02-19 23:48:27 -080038 return glue_fpu_begin(CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS,
39 walk, fpu_enabled, nbytes);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020040}
41
42static inline void cast5_fpu_end(bool fpu_enabled)
43{
44 return glue_fpu_end(fpu_enabled);
45}
46
Eric Biggers1e631832018-02-19 23:48:13 -080047static int ecb_crypt(struct skcipher_request *req, bool enc)
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020048{
49 bool fpu_enabled = false;
Eric Biggers1e631832018-02-19 23:48:13 -080050 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
51 struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
52 struct skcipher_walk walk;
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020053 const unsigned int bsize = CAST5_BLOCK_SIZE;
54 unsigned int nbytes;
Jussi Kivilinnac12ab202012-10-20 15:06:56 +030055 void (*fn)(struct cast5_ctx *ctx, u8 *dst, const u8 *src);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020056 int err;
57
Eric Biggers1e631832018-02-19 23:48:13 -080058 err = skcipher_walk_virt(&walk, req, false);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020059
Eric Biggers1e631832018-02-19 23:48:13 -080060 while ((nbytes = walk.nbytes)) {
61 u8 *wsrc = walk.src.virt.addr;
62 u8 *wdst = walk.dst.virt.addr;
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020063
Eric Biggers1e631832018-02-19 23:48:13 -080064 fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020065
66 /* Process multi-block batch */
67 if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
Eric Biggers8f461b12018-02-19 23:48:12 -080068 fn = (enc) ? cast5_ecb_enc_16way : cast5_ecb_dec_16way;
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020069 do {
Jussi Kivilinnac12ab202012-10-20 15:06:56 +030070 fn(ctx, wdst, wsrc);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020071
72 wsrc += bsize * CAST5_PARALLEL_BLOCKS;
73 wdst += bsize * CAST5_PARALLEL_BLOCKS;
74 nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
75 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
76
77 if (nbytes < bsize)
78 goto done;
79 }
80
Jussi Kivilinnac12ab202012-10-20 15:06:56 +030081 fn = (enc) ? __cast5_encrypt : __cast5_decrypt;
82
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020083 /* Handle leftovers */
84 do {
Jussi Kivilinnac12ab202012-10-20 15:06:56 +030085 fn(ctx, wdst, wsrc);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020086
87 wsrc += bsize;
88 wdst += bsize;
89 nbytes -= bsize;
90 } while (nbytes >= bsize);
91
92done:
Eric Biggers1e631832018-02-19 23:48:13 -080093 err = skcipher_walk_done(&walk, nbytes);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +020094 }
95
96 cast5_fpu_end(fpu_enabled);
97 return err;
98}
99
Eric Biggers1e631832018-02-19 23:48:13 -0800100static int ecb_encrypt(struct skcipher_request *req)
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200101{
Eric Biggers1e631832018-02-19 23:48:13 -0800102 return ecb_crypt(req, true);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200103}
104
Eric Biggers1e631832018-02-19 23:48:13 -0800105static int ecb_decrypt(struct skcipher_request *req)
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200106{
Eric Biggers1e631832018-02-19 23:48:13 -0800107 return ecb_crypt(req, false);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200108}
109
Eric Biggers1e631832018-02-19 23:48:13 -0800110static int cbc_encrypt(struct skcipher_request *req)
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200111{
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200112 const unsigned int bsize = CAST5_BLOCK_SIZE;
Eric Biggers1e631832018-02-19 23:48:13 -0800113 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
114 struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
115 struct skcipher_walk walk;
116 unsigned int nbytes;
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200117 int err;
118
Eric Biggers1e631832018-02-19 23:48:13 -0800119 err = skcipher_walk_virt(&walk, req, false);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200120
121 while ((nbytes = walk.nbytes)) {
Eric Biggers1e631832018-02-19 23:48:13 -0800122 u64 *src = (u64 *)walk.src.virt.addr;
123 u64 *dst = (u64 *)walk.dst.virt.addr;
124 u64 *iv = (u64 *)walk.iv;
125
126 do {
127 *dst = *src ^ *iv;
128 __cast5_encrypt(ctx, (u8 *)dst, (u8 *)dst);
129 iv = dst;
130 src++;
131 dst++;
132 nbytes -= bsize;
133 } while (nbytes >= bsize);
134
135 *(u64 *)walk.iv = *iv;
136 err = skcipher_walk_done(&walk, nbytes);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200137 }
138
139 return err;
140}
141
Eric Biggers1e631832018-02-19 23:48:13 -0800142static unsigned int __cbc_decrypt(struct cast5_ctx *ctx,
143 struct skcipher_walk *walk)
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200144{
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200145 const unsigned int bsize = CAST5_BLOCK_SIZE;
146 unsigned int nbytes = walk->nbytes;
147 u64 *src = (u64 *)walk->src.virt.addr;
148 u64 *dst = (u64 *)walk->dst.virt.addr;
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200149 u64 last_iv;
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200150
151 /* Start of the last block. */
152 src += nbytes / bsize - 1;
153 dst += nbytes / bsize - 1;
154
155 last_iv = *src;
156
157 /* Process multi-block batch */
158 if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
159 do {
160 nbytes -= bsize * (CAST5_PARALLEL_BLOCKS - 1);
161 src -= CAST5_PARALLEL_BLOCKS - 1;
162 dst -= CAST5_PARALLEL_BLOCKS - 1;
163
Jussi Kivilinnac12ab202012-10-20 15:06:56 +0300164 cast5_cbc_dec_16way(ctx, (u8 *)dst, (u8 *)src);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200165
166 nbytes -= bsize;
167 if (nbytes < bsize)
168 goto done;
169
170 *dst ^= *(src - 1);
171 src -= 1;
172 dst -= 1;
173 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200174 }
175
176 /* Handle leftovers */
177 for (;;) {
178 __cast5_decrypt(ctx, (u8 *)dst, (u8 *)src);
179
180 nbytes -= bsize;
181 if (nbytes < bsize)
182 break;
183
184 *dst ^= *(src - 1);
185 src -= 1;
186 dst -= 1;
187 }
188
189done:
190 *dst ^= *(u64 *)walk->iv;
191 *(u64 *)walk->iv = last_iv;
192
193 return nbytes;
194}
195
Eric Biggers1e631832018-02-19 23:48:13 -0800196static int cbc_decrypt(struct skcipher_request *req)
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200197{
Eric Biggers1e631832018-02-19 23:48:13 -0800198 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
199 struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200200 bool fpu_enabled = false;
Eric Biggers1e631832018-02-19 23:48:13 -0800201 struct skcipher_walk walk;
202 unsigned int nbytes;
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200203 int err;
204
Eric Biggers1e631832018-02-19 23:48:13 -0800205 err = skcipher_walk_virt(&walk, req, false);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200206
207 while ((nbytes = walk.nbytes)) {
Eric Biggers1e631832018-02-19 23:48:13 -0800208 fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes);
209 nbytes = __cbc_decrypt(ctx, &walk);
210 err = skcipher_walk_done(&walk, nbytes);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200211 }
212
213 cast5_fpu_end(fpu_enabled);
214 return err;
215}
216
Eric Biggers1e631832018-02-19 23:48:13 -0800217static void ctr_crypt_final(struct skcipher_walk *walk, struct cast5_ctx *ctx)
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200218{
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200219 u8 *ctrblk = walk->iv;
220 u8 keystream[CAST5_BLOCK_SIZE];
221 u8 *src = walk->src.virt.addr;
222 u8 *dst = walk->dst.virt.addr;
223 unsigned int nbytes = walk->nbytes;
224
225 __cast5_encrypt(ctx, keystream, ctrblk);
Ard Biesheuvel45fe93d2017-07-24 11:28:04 +0100226 crypto_xor_cpy(dst, keystream, src, nbytes);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200227
228 crypto_inc(ctrblk, CAST5_BLOCK_SIZE);
229}
230
Eric Biggers1e631832018-02-19 23:48:13 -0800231static unsigned int __ctr_crypt(struct skcipher_walk *walk,
232 struct cast5_ctx *ctx)
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200233{
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200234 const unsigned int bsize = CAST5_BLOCK_SIZE;
235 unsigned int nbytes = walk->nbytes;
236 u64 *src = (u64 *)walk->src.virt.addr;
237 u64 *dst = (u64 *)walk->dst.virt.addr;
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200238
239 /* Process multi-block batch */
240 if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
241 do {
Jussi Kivilinnac12ab202012-10-20 15:06:56 +0300242 cast5_ctr_16way(ctx, (u8 *)dst, (u8 *)src,
243 (__be64 *)walk->iv);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200244
245 src += CAST5_PARALLEL_BLOCKS;
246 dst += CAST5_PARALLEL_BLOCKS;
247 nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
248 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
249
250 if (nbytes < bsize)
251 goto done;
252 }
253
254 /* Handle leftovers */
255 do {
Jussi Kivilinnac12ab202012-10-20 15:06:56 +0300256 u64 ctrblk;
257
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200258 if (dst != src)
259 *dst = *src;
260
Jussi Kivilinnac12ab202012-10-20 15:06:56 +0300261 ctrblk = *(u64 *)walk->iv;
262 be64_add_cpu((__be64 *)walk->iv, 1);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200263
Jussi Kivilinnac12ab202012-10-20 15:06:56 +0300264 __cast5_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
265 *dst ^= ctrblk;
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200266
267 src += 1;
268 dst += 1;
269 nbytes -= bsize;
270 } while (nbytes >= bsize);
271
272done:
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200273 return nbytes;
274}
275
Eric Biggers1e631832018-02-19 23:48:13 -0800276static int ctr_crypt(struct skcipher_request *req)
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200277{
Eric Biggers1e631832018-02-19 23:48:13 -0800278 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
279 struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200280 bool fpu_enabled = false;
Eric Biggers1e631832018-02-19 23:48:13 -0800281 struct skcipher_walk walk;
282 unsigned int nbytes;
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200283 int err;
284
Eric Biggers1e631832018-02-19 23:48:13 -0800285 err = skcipher_walk_virt(&walk, req, false);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200286
287 while ((nbytes = walk.nbytes) >= CAST5_BLOCK_SIZE) {
Eric Biggers1e631832018-02-19 23:48:13 -0800288 fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes);
289 nbytes = __ctr_crypt(&walk, ctx);
290 err = skcipher_walk_done(&walk, nbytes);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200291 }
292
293 cast5_fpu_end(fpu_enabled);
294
295 if (walk.nbytes) {
Eric Biggers1e631832018-02-19 23:48:13 -0800296 ctr_crypt_final(&walk, ctx);
297 err = skcipher_walk_done(&walk, 0);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200298 }
299
300 return err;
301}
302
Eric Biggers1e631832018-02-19 23:48:13 -0800303static struct skcipher_alg cast5_algs[] = {
304 {
305 .base.cra_name = "__ecb(cast5)",
306 .base.cra_driver_name = "__ecb-cast5-avx",
307 .base.cra_priority = 200,
308 .base.cra_flags = CRYPTO_ALG_INTERNAL,
309 .base.cra_blocksize = CAST5_BLOCK_SIZE,
310 .base.cra_ctxsize = sizeof(struct cast5_ctx),
311 .base.cra_module = THIS_MODULE,
312 .min_keysize = CAST5_MIN_KEY_SIZE,
313 .max_keysize = CAST5_MAX_KEY_SIZE,
314 .setkey = cast5_setkey_skcipher,
315 .encrypt = ecb_encrypt,
316 .decrypt = ecb_decrypt,
317 }, {
318 .base.cra_name = "__cbc(cast5)",
319 .base.cra_driver_name = "__cbc-cast5-avx",
320 .base.cra_priority = 200,
321 .base.cra_flags = CRYPTO_ALG_INTERNAL,
322 .base.cra_blocksize = CAST5_BLOCK_SIZE,
323 .base.cra_ctxsize = sizeof(struct cast5_ctx),
324 .base.cra_module = THIS_MODULE,
325 .min_keysize = CAST5_MIN_KEY_SIZE,
326 .max_keysize = CAST5_MAX_KEY_SIZE,
327 .ivsize = CAST5_BLOCK_SIZE,
328 .setkey = cast5_setkey_skcipher,
329 .encrypt = cbc_encrypt,
330 .decrypt = cbc_decrypt,
331 }, {
332 .base.cra_name = "__ctr(cast5)",
333 .base.cra_driver_name = "__ctr-cast5-avx",
334 .base.cra_priority = 200,
335 .base.cra_flags = CRYPTO_ALG_INTERNAL,
336 .base.cra_blocksize = 1,
337 .base.cra_ctxsize = sizeof(struct cast5_ctx),
338 .base.cra_module = THIS_MODULE,
339 .min_keysize = CAST5_MIN_KEY_SIZE,
340 .max_keysize = CAST5_MAX_KEY_SIZE,
341 .ivsize = CAST5_BLOCK_SIZE,
342 .chunksize = CAST5_BLOCK_SIZE,
343 .setkey = cast5_setkey_skcipher,
344 .encrypt = ctr_crypt,
345 .decrypt = ctr_crypt,
346 }
347};
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200348
Eric Biggers1e631832018-02-19 23:48:13 -0800349static struct simd_skcipher_alg *cast5_simd_algs[ARRAY_SIZE(cast5_algs)];
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200350
351static int __init cast5_init(void)
352{
Ingo Molnard5d34d92015-04-28 10:11:24 +0200353 const char *feature_name;
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200354
Dave Hansend91cab72015-09-02 16:31:26 -0700355 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
356 &feature_name)) {
Ingo Molnard5d34d92015-04-28 10:11:24 +0200357 pr_info("CPU feature '%s' is not supported.\n", feature_name);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200358 return -ENODEV;
359 }
360
Eric Biggers1e631832018-02-19 23:48:13 -0800361 return simd_register_skciphers_compat(cast5_algs,
362 ARRAY_SIZE(cast5_algs),
363 cast5_simd_algs);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200364}
365
366static void __exit cast5_exit(void)
367{
Eric Biggers1e631832018-02-19 23:48:13 -0800368 simd_unregister_skciphers(cast5_algs, ARRAY_SIZE(cast5_algs),
369 cast5_simd_algs);
Johannes Goetzfried4d6d6a22012-07-11 19:37:37 +0200370}
371
372module_init(cast5_init);
373module_exit(cast5_exit);
374
375MODULE_DESCRIPTION("Cast5 Cipher Algorithm, AVX optimized");
376MODULE_LICENSE("GPL");
Kees Cook5d26a102014-11-20 17:05:53 -0800377MODULE_ALIAS_CRYPTO("cast5");