Thomas Gleixner | d2912cb | 2019-06-04 10:11:33 +0200 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 2 | /* |
| 3 | * GCM: Galois/Counter Mode. |
| 4 | * |
| 5 | * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 6 | */ |
| 7 | |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 8 | #include <crypto/gf128mul.h> |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 9 | #include <crypto/internal/aead.h> |
Herbert Xu | 1472e5e | 2007-12-07 19:26:11 +0800 | [diff] [blame] | 10 | #include <crypto/internal/skcipher.h> |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 11 | #include <crypto/internal/hash.h> |
Herbert Xu | 17db854 | 2015-05-21 15:11:11 +0800 | [diff] [blame] | 12 | #include <crypto/null.h> |
Herbert Xu | 42c271c | 2007-12-07 18:52:49 +0800 | [diff] [blame] | 13 | #include <crypto/scatterwalk.h> |
Corentin LABBE | e0ab7e9 | 2017-08-22 10:08:17 +0200 | [diff] [blame] | 14 | #include <crypto/gcm.h> |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 15 | #include <crypto/hash.h> |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 16 | #include <linux/err.h> |
| 17 | #include <linux/init.h> |
| 18 | #include <linux/kernel.h> |
| 19 | #include <linux/module.h> |
| 20 | #include <linux/slab.h> |
| 21 | |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 22 | struct gcm_instance_ctx { |
Herbert Xu | 1472e5e | 2007-12-07 19:26:11 +0800 | [diff] [blame] | 23 | struct crypto_skcipher_spawn ctr; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 24 | struct crypto_ahash_spawn ghash; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 25 | }; |
| 26 | |
| 27 | struct crypto_gcm_ctx { |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 28 | struct crypto_skcipher *ctr; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 29 | struct crypto_ahash *ghash; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 30 | }; |
| 31 | |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 32 | struct crypto_rfc4106_ctx { |
| 33 | struct crypto_aead *child; |
| 34 | u8 nonce[4]; |
| 35 | }; |
| 36 | |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 37 | struct crypto_rfc4106_req_ctx { |
| 38 | struct scatterlist src[3]; |
| 39 | struct scatterlist dst[3]; |
| 40 | struct aead_request subreq; |
| 41 | }; |
| 42 | |
Jussi Kivilinna | 9489667d | 2013-04-07 16:43:41 +0300 | [diff] [blame] | 43 | struct crypto_rfc4543_instance_ctx { |
| 44 | struct crypto_aead_spawn aead; |
Jussi Kivilinna | 9489667d | 2013-04-07 16:43:41 +0300 | [diff] [blame] | 45 | }; |
| 46 | |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 47 | struct crypto_rfc4543_ctx { |
| 48 | struct crypto_aead *child; |
Kees Cook | 8d60539 | 2018-09-18 19:10:51 -0700 | [diff] [blame] | 49 | struct crypto_sync_skcipher *null; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 50 | u8 nonce[4]; |
| 51 | }; |
| 52 | |
| 53 | struct crypto_rfc4543_req_ctx { |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 54 | struct aead_request subreq; |
| 55 | }; |
| 56 | |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 57 | struct crypto_gcm_ghash_ctx { |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 58 | unsigned int cryptlen; |
| 59 | struct scatterlist *src; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 60 | int (*complete)(struct aead_request *req, u32 flags); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 61 | }; |
| 62 | |
| 63 | struct crypto_gcm_req_priv_ctx { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 64 | u8 iv[16]; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 65 | u8 auth_tag[16]; |
Herbert Xu | 6160b28 | 2007-12-04 19:17:50 +1100 | [diff] [blame] | 66 | u8 iauth_tag[16]; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 67 | struct scatterlist src[3]; |
| 68 | struct scatterlist dst[3]; |
| 69 | struct scatterlist sg; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 70 | struct crypto_gcm_ghash_ctx ghash_ctx; |
| 71 | union { |
| 72 | struct ahash_request ahreq; |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 73 | struct skcipher_request skreq; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 74 | } u; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 75 | }; |
| 76 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 77 | static struct { |
| 78 | u8 buf[16]; |
| 79 | struct scatterlist sg; |
| 80 | } *gcm_zeroes; |
| 81 | |
| 82 | static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 83 | |
Herbert Xu | 2589469 | 2007-12-12 19:16:38 +0800 | [diff] [blame] | 84 | static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( |
| 85 | struct aead_request *req) |
| 86 | { |
| 87 | unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req)); |
| 88 | |
| 89 | return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); |
| 90 | } |
| 91 | |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 92 | static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, |
| 93 | unsigned int keylen) |
| 94 | { |
| 95 | struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 96 | struct crypto_ahash *ghash = ctx->ghash; |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 97 | struct crypto_skcipher *ctr = ctx->ctr; |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 98 | struct { |
| 99 | be128 hash; |
Ondrej Mosnáček | 50d2e6d | 2016-09-23 10:47:32 +0200 | [diff] [blame] | 100 | u8 iv[16]; |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 101 | |
Gilad Ben-Yossef | 76c6739 | 2017-10-18 08:00:42 +0100 | [diff] [blame] | 102 | struct crypto_wait wait; |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 103 | |
| 104 | struct scatterlist sg[1]; |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 105 | struct skcipher_request req; |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 106 | } *data; |
| 107 | int err; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 108 | |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 109 | crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); |
| 110 | crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) & |
| 111 | CRYPTO_TFM_REQ_MASK); |
| 112 | err = crypto_skcipher_setkey(ctr, key, keylen); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 113 | if (err) |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 114 | return err; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 115 | |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 116 | data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr), |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 117 | GFP_KERNEL); |
| 118 | if (!data) |
| 119 | return -ENOMEM; |
| 120 | |
Gilad Ben-Yossef | 76c6739 | 2017-10-18 08:00:42 +0100 | [diff] [blame] | 121 | crypto_init_wait(&data->wait); |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 122 | sg_init_one(data->sg, &data->hash, sizeof(data->hash)); |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 123 | skcipher_request_set_tfm(&data->req, ctr); |
| 124 | skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | |
| 125 | CRYPTO_TFM_REQ_MAY_BACKLOG, |
Gilad Ben-Yossef | 76c6739 | 2017-10-18 08:00:42 +0100 | [diff] [blame] | 126 | crypto_req_done, |
| 127 | &data->wait); |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 128 | skcipher_request_set_crypt(&data->req, data->sg, data->sg, |
| 129 | sizeof(data->hash), data->iv); |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 130 | |
Gilad Ben-Yossef | 76c6739 | 2017-10-18 08:00:42 +0100 | [diff] [blame] | 131 | err = crypto_wait_req(crypto_skcipher_encrypt(&data->req), |
| 132 | &data->wait); |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 133 | |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 134 | if (err) |
| 135 | goto out; |
| 136 | |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 137 | crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); |
| 138 | crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & |
| 139 | CRYPTO_TFM_REQ_MASK); |
| 140 | err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 141 | out: |
Waiman Long | 453431a | 2020-08-06 23:18:13 -0700 | [diff] [blame] | 142 | kfree_sensitive(data); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 143 | return err; |
| 144 | } |
| 145 | |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 146 | static int crypto_gcm_setauthsize(struct crypto_aead *tfm, |
| 147 | unsigned int authsize) |
| 148 | { |
Iuliana Prodan | 65526f6 | 2019-07-31 16:05:54 +0300 | [diff] [blame] | 149 | return crypto_gcm_check_authsize(authsize); |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 150 | } |
| 151 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 152 | static void crypto_gcm_init_common(struct aead_request *req) |
| 153 | { |
| 154 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
| 155 | __be32 counter = cpu_to_be32(1); |
| 156 | struct scatterlist *sg; |
| 157 | |
| 158 | memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag)); |
Corentin LABBE | e0ab7e9 | 2017-08-22 10:08:17 +0200 | [diff] [blame] | 159 | memcpy(pctx->iv, req->iv, GCM_AES_IV_SIZE); |
| 160 | memcpy(pctx->iv + GCM_AES_IV_SIZE, &counter, 4); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 161 | |
| 162 | sg_init_table(pctx->src, 3); |
| 163 | sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag)); |
| 164 | sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen); |
| 165 | if (sg != pctx->src + 1) |
Dan Williams | c56f6d1 | 2015-08-07 18:15:13 +0200 | [diff] [blame] | 166 | sg_chain(pctx->src, 2, sg); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 167 | |
| 168 | if (req->src != req->dst) { |
| 169 | sg_init_table(pctx->dst, 3); |
| 170 | sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag)); |
| 171 | sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen); |
| 172 | if (sg != pctx->dst + 1) |
Dan Williams | c56f6d1 | 2015-08-07 18:15:13 +0200 | [diff] [blame] | 173 | sg_chain(pctx->dst, 2, sg); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 174 | } |
| 175 | } |
| 176 | |
| 177 | static void crypto_gcm_init_crypt(struct aead_request *req, |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 178 | unsigned int cryptlen) |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 179 | { |
| 180 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
| 181 | struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); |
Herbert Xu | 2589469 | 2007-12-12 19:16:38 +0800 | [diff] [blame] | 182 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 183 | struct skcipher_request *skreq = &pctx->u.skreq; |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 184 | struct scatterlist *dst; |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 185 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 186 | dst = req->src == req->dst ? pctx->src : pctx->dst; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 187 | |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 188 | skcipher_request_set_tfm(skreq, ctx->ctr); |
| 189 | skcipher_request_set_crypt(skreq, pctx->src, dst, |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 190 | cryptlen + sizeof(pctx->auth_tag), |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 191 | pctx->iv); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 192 | } |
| 193 | |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 194 | static inline unsigned int gcm_remain(unsigned int len) |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 195 | { |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 196 | len &= 0xfU; |
| 197 | return len ? 16 - len : 0; |
| 198 | } |
| 199 | |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 200 | static void gcm_hash_len_done(void *data, int err); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 201 | |
| 202 | static int gcm_hash_update(struct aead_request *req, |
Mark Rustad | 3e3dc25 | 2014-07-25 02:53:38 -0700 | [diff] [blame] | 203 | crypto_completion_t compl, |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 204 | struct scatterlist *src, |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 205 | unsigned int len, u32 flags) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 206 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 207 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 208 | struct ahash_request *ahreq = &pctx->u.ahreq; |
| 209 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 210 | ahash_request_set_callback(ahreq, flags, compl, req); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 211 | ahash_request_set_crypt(ahreq, src, NULL, len); |
| 212 | |
| 213 | return crypto_ahash_update(ahreq); |
| 214 | } |
| 215 | |
| 216 | static int gcm_hash_remain(struct aead_request *req, |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 217 | unsigned int remain, |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 218 | crypto_completion_t compl, u32 flags) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 219 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 220 | return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 221 | } |
| 222 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 223 | static int gcm_hash_len(struct aead_request *req, u32 flags) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 224 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 225 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 226 | struct ahash_request *ahreq = &pctx->u.ahreq; |
| 227 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; |
Eric Biggers | 1866655 | 2019-01-10 12:17:52 -0800 | [diff] [blame] | 228 | be128 lengths; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 229 | |
| 230 | lengths.a = cpu_to_be64(req->assoclen * 8); |
| 231 | lengths.b = cpu_to_be64(gctx->cryptlen * 8); |
| 232 | memcpy(pctx->iauth_tag, &lengths, 16); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 233 | sg_init_one(&pctx->sg, pctx->iauth_tag, 16); |
| 234 | ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req); |
| 235 | ahash_request_set_crypt(ahreq, &pctx->sg, |
| 236 | pctx->iauth_tag, sizeof(lengths)); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 237 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 238 | return crypto_ahash_finup(ahreq); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 239 | } |
| 240 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 241 | static int gcm_hash_len_continue(struct aead_request *req, u32 flags) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 242 | { |
Herbert Xu | 2589469 | 2007-12-12 19:16:38 +0800 | [diff] [blame] | 243 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 244 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 245 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 246 | return gctx->complete(req, flags); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 247 | } |
| 248 | |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 249 | static void gcm_hash_len_done(void *data, int err) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 250 | { |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 251 | struct aead_request *req = data; |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 252 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 253 | if (err) |
| 254 | goto out; |
| 255 | |
| 256 | err = gcm_hash_len_continue(req, 0); |
| 257 | if (err == -EINPROGRESS) |
| 258 | return; |
| 259 | |
| 260 | out: |
| 261 | aead_request_complete(req, err); |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 262 | } |
| 263 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 264 | static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags) |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 265 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 266 | return gcm_hash_len(req, flags) ?: |
| 267 | gcm_hash_len_continue(req, flags); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 268 | } |
| 269 | |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 270 | static void gcm_hash_crypt_remain_done(void *data, int err) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 271 | { |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 272 | struct aead_request *req = data; |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 273 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 274 | if (err) |
| 275 | goto out; |
| 276 | |
| 277 | err = gcm_hash_crypt_remain_continue(req, 0); |
| 278 | if (err == -EINPROGRESS) |
| 279 | return; |
| 280 | |
| 281 | out: |
| 282 | aead_request_complete(req, err); |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 283 | } |
| 284 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 285 | static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags) |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 286 | { |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 287 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
| 288 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; |
| 289 | unsigned int remain; |
| 290 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 291 | remain = gcm_remain(gctx->cryptlen); |
| 292 | if (remain) |
| 293 | return gcm_hash_remain(req, remain, |
| 294 | gcm_hash_crypt_remain_done, flags) ?: |
| 295 | gcm_hash_crypt_remain_continue(req, flags); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 296 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 297 | return gcm_hash_crypt_remain_continue(req, flags); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 298 | } |
| 299 | |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 300 | static void gcm_hash_crypt_done(void *data, int err) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 301 | { |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 302 | struct aead_request *req = data; |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 303 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 304 | if (err) |
| 305 | goto out; |
| 306 | |
| 307 | err = gcm_hash_crypt_continue(req, 0); |
| 308 | if (err == -EINPROGRESS) |
| 309 | return; |
| 310 | |
| 311 | out: |
| 312 | aead_request_complete(req, err); |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 313 | } |
| 314 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 315 | static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags) |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 316 | { |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 317 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
| 318 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 319 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 320 | if (gctx->cryptlen) |
| 321 | return gcm_hash_update(req, gcm_hash_crypt_done, |
| 322 | gctx->src, gctx->cryptlen, flags) ?: |
| 323 | gcm_hash_crypt_continue(req, flags); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 324 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 325 | return gcm_hash_crypt_remain_continue(req, flags); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 326 | } |
| 327 | |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 328 | static void gcm_hash_assoc_remain_done(void *data, int err) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 329 | { |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 330 | struct aead_request *req = data; |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 331 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 332 | if (err) |
| 333 | goto out; |
| 334 | |
| 335 | err = gcm_hash_assoc_remain_continue(req, 0); |
| 336 | if (err == -EINPROGRESS) |
| 337 | return; |
| 338 | |
| 339 | out: |
| 340 | aead_request_complete(req, err); |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 341 | } |
| 342 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 343 | static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags) |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 344 | { |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 345 | unsigned int remain; |
| 346 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 347 | remain = gcm_remain(req->assoclen); |
| 348 | if (remain) |
| 349 | return gcm_hash_remain(req, remain, |
| 350 | gcm_hash_assoc_remain_done, flags) ?: |
| 351 | gcm_hash_assoc_remain_continue(req, flags); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 352 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 353 | return gcm_hash_assoc_remain_continue(req, flags); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 354 | } |
| 355 | |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 356 | static void gcm_hash_assoc_done(void *data, int err) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 357 | { |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 358 | struct aead_request *req = data; |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 359 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 360 | if (err) |
| 361 | goto out; |
| 362 | |
| 363 | err = gcm_hash_assoc_continue(req, 0); |
| 364 | if (err == -EINPROGRESS) |
| 365 | return; |
| 366 | |
| 367 | out: |
| 368 | aead_request_complete(req, err); |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 369 | } |
| 370 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 371 | static int gcm_hash_init_continue(struct aead_request *req, u32 flags) |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 372 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 373 | if (req->assoclen) |
| 374 | return gcm_hash_update(req, gcm_hash_assoc_done, |
| 375 | req->src, req->assoclen, flags) ?: |
| 376 | gcm_hash_assoc_continue(req, flags); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 377 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 378 | return gcm_hash_assoc_remain_continue(req, flags); |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 379 | } |
| 380 | |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 381 | static void gcm_hash_init_done(void *data, int err) |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 382 | { |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 383 | struct aead_request *req = data; |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 384 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 385 | if (err) |
| 386 | goto out; |
| 387 | |
| 388 | err = gcm_hash_init_continue(req, 0); |
| 389 | if (err == -EINPROGRESS) |
| 390 | return; |
| 391 | |
| 392 | out: |
| 393 | aead_request_complete(req, err); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 394 | } |
| 395 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 396 | static int gcm_hash(struct aead_request *req, u32 flags) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 397 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 398 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 399 | struct ahash_request *ahreq = &pctx->u.ahreq; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 400 | struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 401 | |
| 402 | ahash_request_set_tfm(ahreq, ctx->ghash); |
| 403 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 404 | ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req); |
| 405 | return crypto_ahash_init(ahreq) ?: |
| 406 | gcm_hash_init_continue(req, flags); |
Herbert Xu | 6160b28 | 2007-12-04 19:17:50 +1100 | [diff] [blame] | 407 | } |
| 408 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 409 | static int gcm_enc_copy_hash(struct aead_request *req, u32 flags) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 410 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 411 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 412 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
| 413 | u8 *auth_tag = pctx->auth_tag; |
| 414 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 415 | crypto_xor(auth_tag, pctx->iauth_tag, 16); |
| 416 | scatterwalk_map_and_copy(auth_tag, req->dst, |
| 417 | req->assoclen + req->cryptlen, |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 418 | crypto_aead_authsize(aead), 1); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 419 | return 0; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 420 | } |
| 421 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 422 | static int gcm_encrypt_continue(struct aead_request *req, u32 flags) |
Herbert Xu | 6160b28 | 2007-12-04 19:17:50 +1100 | [diff] [blame] | 423 | { |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 424 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 425 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; |
Herbert Xu | 6160b28 | 2007-12-04 19:17:50 +1100 | [diff] [blame] | 426 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 427 | gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst); |
| 428 | gctx->cryptlen = req->cryptlen; |
| 429 | gctx->complete = gcm_enc_copy_hash; |
Herbert Xu | 6160b28 | 2007-12-04 19:17:50 +1100 | [diff] [blame] | 430 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 431 | return gcm_hash(req, flags); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 432 | } |
| 433 | |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 434 | static void gcm_encrypt_done(void *data, int err) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 435 | { |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 436 | struct aead_request *req = data; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 437 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 438 | if (err) |
| 439 | goto out; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 440 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 441 | err = gcm_encrypt_continue(req, 0); |
| 442 | if (err == -EINPROGRESS) |
| 443 | return; |
| 444 | |
| 445 | out: |
Huang Ying | 62c5593 | 2009-11-16 21:52:22 +0800 | [diff] [blame] | 446 | aead_request_complete(req, err); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 447 | } |
| 448 | |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 449 | static int crypto_gcm_encrypt(struct aead_request *req) |
| 450 | { |
Herbert Xu | 2589469 | 2007-12-12 19:16:38 +0800 | [diff] [blame] | 451 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 452 | struct skcipher_request *skreq = &pctx->u.skreq; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 453 | u32 flags = aead_request_flags(req); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 454 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 455 | crypto_gcm_init_common(req); |
| 456 | crypto_gcm_init_crypt(req, req->cryptlen); |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 457 | skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 458 | |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 459 | return crypto_skcipher_encrypt(skreq) ?: |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 460 | gcm_encrypt_continue(req, flags); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 461 | } |
| 462 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 463 | static int crypto_gcm_verify(struct aead_request *req) |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 464 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 465 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 466 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 467 | u8 *auth_tag = pctx->auth_tag; |
| 468 | u8 *iauth_tag = pctx->iauth_tag; |
| 469 | unsigned int authsize = crypto_aead_authsize(aead); |
| 470 | unsigned int cryptlen = req->cryptlen - authsize; |
| 471 | |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 472 | crypto_xor(auth_tag, iauth_tag, 16); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 473 | scatterwalk_map_and_copy(iauth_tag, req->src, |
| 474 | req->assoclen + cryptlen, authsize, 0); |
James Yonan | 6bf37e5 | 2013-09-26 02:20:39 -0600 | [diff] [blame] | 475 | return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 476 | } |
| 477 | |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 478 | static void gcm_decrypt_done(void *data, int err) |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 479 | { |
Herbert Xu | 255e48e | 2023-02-08 13:58:44 +0800 | [diff] [blame] | 480 | struct aead_request *req = data; |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 481 | |
| 482 | if (!err) |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 483 | err = crypto_gcm_verify(req); |
Herbert Xu | 84c9115 | 2007-12-17 21:42:08 +0800 | [diff] [blame] | 484 | |
| 485 | aead_request_complete(req, err); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 486 | } |
| 487 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 488 | static int gcm_dec_hash_continue(struct aead_request *req, u32 flags) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 489 | { |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 490 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 491 | struct skcipher_request *skreq = &pctx->u.skreq; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 492 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; |
| 493 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 494 | crypto_gcm_init_crypt(req, gctx->cryptlen); |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 495 | skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req); |
| 496 | return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 497 | } |
| 498 | |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 499 | static int crypto_gcm_decrypt(struct aead_request *req) |
| 500 | { |
Herbert Xu | 6160b28 | 2007-12-04 19:17:50 +1100 | [diff] [blame] | 501 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
Herbert Xu | 2589469 | 2007-12-12 19:16:38 +0800 | [diff] [blame] | 502 | struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 503 | struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; |
Herbert Xu | 6160b28 | 2007-12-04 19:17:50 +1100 | [diff] [blame] | 504 | unsigned int authsize = crypto_aead_authsize(aead); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 505 | unsigned int cryptlen = req->cryptlen; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 506 | u32 flags = aead_request_flags(req); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 507 | |
Herbert Xu | 6160b28 | 2007-12-04 19:17:50 +1100 | [diff] [blame] | 508 | cryptlen -= authsize; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 509 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 510 | crypto_gcm_init_common(req); |
| 511 | |
| 512 | gctx->src = sg_next(pctx->src); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 513 | gctx->cryptlen = cryptlen; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 514 | gctx->complete = gcm_dec_hash_continue; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 515 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 516 | return gcm_hash(req, flags); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 517 | } |
| 518 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 519 | static int crypto_gcm_init_tfm(struct crypto_aead *tfm) |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 520 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 521 | struct aead_instance *inst = aead_alg_instance(tfm); |
| 522 | struct gcm_instance_ctx *ictx = aead_instance_ctx(inst); |
| 523 | struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 524 | struct crypto_skcipher *ctr; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 525 | struct crypto_ahash *ghash; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 526 | unsigned long align; |
| 527 | int err; |
| 528 | |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 529 | ghash = crypto_spawn_ahash(&ictx->ghash); |
| 530 | if (IS_ERR(ghash)) |
| 531 | return PTR_ERR(ghash); |
| 532 | |
Eric Biggers | 60425a8 | 2016-10-28 09:52:19 -0700 | [diff] [blame] | 533 | ctr = crypto_spawn_skcipher(&ictx->ctr); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 534 | err = PTR_ERR(ctr); |
| 535 | if (IS_ERR(ctr)) |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 536 | goto err_free_hash; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 537 | |
| 538 | ctx->ctr = ctr; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 539 | ctx->ghash = ghash; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 540 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 541 | align = crypto_aead_alignmask(tfm); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 542 | align &= ~(crypto_tfm_ctx_alignment() - 1); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 543 | crypto_aead_set_reqsize(tfm, |
Herbert Xu | 5d72336 | 2015-05-11 17:47:59 +0800 | [diff] [blame] | 544 | align + offsetof(struct crypto_gcm_req_priv_ctx, u) + |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 545 | max(sizeof(struct skcipher_request) + |
| 546 | crypto_skcipher_reqsize(ctr), |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 547 | sizeof(struct ahash_request) + |
Herbert Xu | 5d72336 | 2015-05-11 17:47:59 +0800 | [diff] [blame] | 548 | crypto_ahash_reqsize(ghash))); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 549 | |
| 550 | return 0; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 551 | |
| 552 | err_free_hash: |
| 553 | crypto_free_ahash(ghash); |
| 554 | return err; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 555 | } |
| 556 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 557 | static void crypto_gcm_exit_tfm(struct crypto_aead *tfm) |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 558 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 559 | struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 560 | |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 561 | crypto_free_ahash(ctx->ghash); |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 562 | crypto_free_skcipher(ctx->ctr); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 563 | } |
| 564 | |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 565 | static void crypto_gcm_free(struct aead_instance *inst) |
| 566 | { |
| 567 | struct gcm_instance_ctx *ctx = aead_instance_ctx(inst); |
| 568 | |
| 569 | crypto_drop_skcipher(&ctx->ctr); |
| 570 | crypto_drop_ahash(&ctx->ghash); |
| 571 | kfree(inst); |
| 572 | } |
| 573 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 574 | static int crypto_gcm_create_common(struct crypto_template *tmpl, |
| 575 | struct rtattr **tb, |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 576 | const char *ctr_name, |
| 577 | const char *ghash_name) |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 578 | { |
Herbert Xu | 712c22a | 2023-10-03 11:43:29 +0800 | [diff] [blame] | 579 | struct skcipher_alg_common *ctr; |
Eric Biggers | b9f76dd | 2020-01-02 19:58:45 -0800 | [diff] [blame] | 580 | u32 mask; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 581 | struct aead_instance *inst; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 582 | struct gcm_instance_ctx *ctx; |
Eric Biggers | ab6ffd3 | 2020-01-02 19:58:57 -0800 | [diff] [blame] | 583 | struct hash_alg_common *ghash; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 584 | int err; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 585 | |
Eric Biggers | 7bcb2c9 | 2020-07-09 23:20:38 -0700 | [diff] [blame] | 586 | err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); |
| 587 | if (err) |
| 588 | return err; |
Eric Biggers | b9f76dd | 2020-01-02 19:58:45 -0800 | [diff] [blame] | 589 | |
Herbert Xu | 1472e5e | 2007-12-07 19:26:11 +0800 | [diff] [blame] | 590 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
| 591 | if (!inst) |
Eric Biggers | ab6ffd3 | 2020-01-02 19:58:57 -0800 | [diff] [blame] | 592 | return -ENOMEM; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 593 | ctx = aead_instance_ctx(inst); |
Eric Biggers | ab6ffd3 | 2020-01-02 19:58:57 -0800 | [diff] [blame] | 594 | |
| 595 | err = crypto_grab_ahash(&ctx->ghash, aead_crypto_instance(inst), |
| 596 | ghash_name, 0, mask); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 597 | if (err) |
| 598 | goto err_free_inst; |
Eric Biggers | ab6ffd3 | 2020-01-02 19:58:57 -0800 | [diff] [blame] | 599 | ghash = crypto_spawn_ahash_alg(&ctx->ghash); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 600 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 601 | err = -EINVAL; |
Eric Biggers | f699594 | 2019-04-18 14:43:02 -0700 | [diff] [blame] | 602 | if (strcmp(ghash->base.cra_name, "ghash") != 0 || |
| 603 | ghash->digestsize != 16) |
Eric Biggers | ab6ffd3 | 2020-01-02 19:58:57 -0800 | [diff] [blame] | 604 | goto err_free_inst; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 605 | |
Eric Biggers | b9f76dd | 2020-01-02 19:58:45 -0800 | [diff] [blame] | 606 | err = crypto_grab_skcipher(&ctx->ctr, aead_crypto_instance(inst), |
| 607 | ctr_name, 0, mask); |
Herbert Xu | 1472e5e | 2007-12-07 19:26:11 +0800 | [diff] [blame] | 608 | if (err) |
Eric Biggers | ab6ffd3 | 2020-01-02 19:58:57 -0800 | [diff] [blame] | 609 | goto err_free_inst; |
Herbert Xu | 712c22a | 2023-10-03 11:43:29 +0800 | [diff] [blame] | 610 | ctr = crypto_spawn_skcipher_alg_common(&ctx->ctr); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 611 | |
Eric Biggers | f699594 | 2019-04-18 14:43:02 -0700 | [diff] [blame] | 612 | /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */ |
Wei Yongjun | 9b40f79 | 2016-10-17 15:10:06 +0000 | [diff] [blame] | 613 | err = -EINVAL; |
Eric Biggers | f699594 | 2019-04-18 14:43:02 -0700 | [diff] [blame] | 614 | if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 || |
Herbert Xu | 712c22a | 2023-10-03 11:43:29 +0800 | [diff] [blame] | 615 | ctr->ivsize != 16 || ctr->base.cra_blocksize != 1) |
Eric Biggers | ab6ffd3 | 2020-01-02 19:58:57 -0800 | [diff] [blame] | 616 | goto err_free_inst; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 617 | |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 618 | err = -ENAMETOOLONG; |
Eric Biggers | f699594 | 2019-04-18 14:43:02 -0700 | [diff] [blame] | 619 | if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, |
| 620 | "gcm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME) |
Eric Biggers | ab6ffd3 | 2020-01-02 19:58:57 -0800 | [diff] [blame] | 621 | goto err_free_inst; |
Eric Biggers | f699594 | 2019-04-18 14:43:02 -0700 | [diff] [blame] | 622 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 623 | if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 624 | "gcm_base(%s,%s)", ctr->base.cra_driver_name, |
Eric Biggers | ab6ffd3 | 2020-01-02 19:58:57 -0800 | [diff] [blame] | 625 | ghash->base.cra_driver_name) >= |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 626 | CRYPTO_MAX_ALG_NAME) |
Eric Biggers | ab6ffd3 | 2020-01-02 19:58:57 -0800 | [diff] [blame] | 627 | goto err_free_inst; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 628 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 629 | inst->alg.base.cra_priority = (ghash->base.cra_priority + |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 630 | ctr->base.cra_priority) / 2; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 631 | inst->alg.base.cra_blocksize = 1; |
Eric Biggers | 33fe2fb | 2023-10-22 01:10:52 -0700 | [diff] [blame] | 632 | inst->alg.base.cra_alignmask = ctr->base.cra_alignmask; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 633 | inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx); |
Corentin LABBE | e0ab7e9 | 2017-08-22 10:08:17 +0200 | [diff] [blame] | 634 | inst->alg.ivsize = GCM_AES_IV_SIZE; |
Herbert Xu | 712c22a | 2023-10-03 11:43:29 +0800 | [diff] [blame] | 635 | inst->alg.chunksize = ctr->chunksize; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 636 | inst->alg.maxauthsize = 16; |
| 637 | inst->alg.init = crypto_gcm_init_tfm; |
| 638 | inst->alg.exit = crypto_gcm_exit_tfm; |
| 639 | inst->alg.setkey = crypto_gcm_setkey; |
| 640 | inst->alg.setauthsize = crypto_gcm_setauthsize; |
| 641 | inst->alg.encrypt = crypto_gcm_encrypt; |
| 642 | inst->alg.decrypt = crypto_gcm_decrypt; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 643 | |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 644 | inst->free = crypto_gcm_free; |
| 645 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 646 | err = aead_register_instance(tmpl, inst); |
Eric Biggers | ab6ffd3 | 2020-01-02 19:58:57 -0800 | [diff] [blame] | 647 | if (err) { |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 648 | err_free_inst: |
Eric Biggers | ab6ffd3 | 2020-01-02 19:58:57 -0800 | [diff] [blame] | 649 | crypto_gcm_free(inst); |
| 650 | } |
| 651 | return err; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 652 | } |
| 653 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 654 | static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 655 | { |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 656 | const char *cipher_name; |
| 657 | char ctr_name[CRYPTO_MAX_ALG_NAME]; |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 658 | |
| 659 | cipher_name = crypto_attr_alg_name(tb[1]); |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 660 | if (IS_ERR(cipher_name)) |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 661 | return PTR_ERR(cipher_name); |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 662 | |
| 663 | if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >= |
| 664 | CRYPTO_MAX_ALG_NAME) |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 665 | return -ENAMETOOLONG; |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 666 | |
Eric Biggers | f699594 | 2019-04-18 14:43:02 -0700 | [diff] [blame] | 667 | return crypto_gcm_create_common(tmpl, tb, ctr_name, "ghash"); |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 668 | } |
| 669 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 670 | static int crypto_gcm_base_create(struct crypto_template *tmpl, |
| 671 | struct rtattr **tb) |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 672 | { |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 673 | const char *ctr_name; |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 674 | const char *ghash_name; |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 675 | |
| 676 | ctr_name = crypto_attr_alg_name(tb[1]); |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 677 | if (IS_ERR(ctr_name)) |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 678 | return PTR_ERR(ctr_name); |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 679 | |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 680 | ghash_name = crypto_attr_alg_name(tb[2]); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 681 | if (IS_ERR(ghash_name)) |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 682 | return PTR_ERR(ghash_name); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 683 | |
Eric Biggers | f699594 | 2019-04-18 14:43:02 -0700 | [diff] [blame] | 684 | return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name); |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 685 | } |
| 686 | |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 687 | static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key, |
| 688 | unsigned int keylen) |
| 689 | { |
| 690 | struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); |
| 691 | struct crypto_aead *child = ctx->child; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 692 | |
| 693 | if (keylen < 4) |
| 694 | return -EINVAL; |
| 695 | |
| 696 | keylen -= 4; |
| 697 | memcpy(ctx->nonce, key + keylen, 4); |
| 698 | |
| 699 | crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); |
| 700 | crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & |
| 701 | CRYPTO_TFM_REQ_MASK); |
Eric Biggers | af5034e | 2019-12-30 21:19:38 -0600 | [diff] [blame] | 702 | return crypto_aead_setkey(child, key, keylen); |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 703 | } |
| 704 | |
| 705 | static int crypto_rfc4106_setauthsize(struct crypto_aead *parent, |
| 706 | unsigned int authsize) |
| 707 | { |
| 708 | struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent); |
Iuliana Prodan | 65526f6 | 2019-07-31 16:05:54 +0300 | [diff] [blame] | 709 | int err; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 710 | |
Iuliana Prodan | 65526f6 | 2019-07-31 16:05:54 +0300 | [diff] [blame] | 711 | err = crypto_rfc4106_check_authsize(authsize); |
| 712 | if (err) |
| 713 | return err; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 714 | |
| 715 | return crypto_aead_setauthsize(ctx->child, authsize); |
| 716 | } |
| 717 | |
| 718 | static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req) |
| 719 | { |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 720 | struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req); |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 721 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
| 722 | struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead); |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 723 | struct aead_request *subreq = &rctx->subreq; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 724 | struct crypto_aead *child = ctx->child; |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 725 | struct scatterlist *sg; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 726 | u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child), |
| 727 | crypto_aead_alignmask(child) + 1); |
| 728 | |
Corentin LABBE | e0ab7e9 | 2017-08-22 10:08:17 +0200 | [diff] [blame] | 729 | scatterwalk_map_and_copy(iv + GCM_AES_IV_SIZE, req->src, 0, req->assoclen - 8, 0); |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 730 | |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 731 | memcpy(iv, ctx->nonce, 4); |
| 732 | memcpy(iv + 4, req->iv, 8); |
| 733 | |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 734 | sg_init_table(rctx->src, 3); |
Corentin LABBE | e0ab7e9 | 2017-08-22 10:08:17 +0200 | [diff] [blame] | 735 | sg_set_buf(rctx->src, iv + GCM_AES_IV_SIZE, req->assoclen - 8); |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 736 | sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen); |
| 737 | if (sg != rctx->src + 1) |
| 738 | sg_chain(rctx->src, 2, sg); |
| 739 | |
| 740 | if (req->src != req->dst) { |
| 741 | sg_init_table(rctx->dst, 3); |
Corentin LABBE | e0ab7e9 | 2017-08-22 10:08:17 +0200 | [diff] [blame] | 742 | sg_set_buf(rctx->dst, iv + GCM_AES_IV_SIZE, req->assoclen - 8); |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 743 | sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen); |
| 744 | if (sg != rctx->dst + 1) |
| 745 | sg_chain(rctx->dst, 2, sg); |
| 746 | } |
| 747 | |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 748 | aead_request_set_tfm(subreq, child); |
| 749 | aead_request_set_callback(subreq, req->base.flags, req->base.complete, |
| 750 | req->base.data); |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 751 | aead_request_set_crypt(subreq, rctx->src, |
| 752 | req->src == req->dst ? rctx->src : rctx->dst, |
| 753 | req->cryptlen, iv); |
| 754 | aead_request_set_ad(subreq, req->assoclen - 8); |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 755 | |
| 756 | return subreq; |
| 757 | } |
| 758 | |
| 759 | static int crypto_rfc4106_encrypt(struct aead_request *req) |
| 760 | { |
Iuliana Prodan | 65526f6 | 2019-07-31 16:05:54 +0300 | [diff] [blame] | 761 | int err; |
| 762 | |
| 763 | err = crypto_ipsec_check_assoclen(req->assoclen); |
| 764 | if (err) |
| 765 | return err; |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 766 | |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 767 | req = crypto_rfc4106_crypt(req); |
| 768 | |
| 769 | return crypto_aead_encrypt(req); |
| 770 | } |
| 771 | |
| 772 | static int crypto_rfc4106_decrypt(struct aead_request *req) |
| 773 | { |
Iuliana Prodan | 65526f6 | 2019-07-31 16:05:54 +0300 | [diff] [blame] | 774 | int err; |
| 775 | |
| 776 | err = crypto_ipsec_check_assoclen(req->assoclen); |
| 777 | if (err) |
| 778 | return err; |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 779 | |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 780 | req = crypto_rfc4106_crypt(req); |
| 781 | |
| 782 | return crypto_aead_decrypt(req); |
| 783 | } |
| 784 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 785 | static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm) |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 786 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 787 | struct aead_instance *inst = aead_alg_instance(tfm); |
| 788 | struct crypto_aead_spawn *spawn = aead_instance_ctx(inst); |
| 789 | struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm); |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 790 | struct crypto_aead *aead; |
| 791 | unsigned long align; |
| 792 | |
| 793 | aead = crypto_spawn_aead(spawn); |
| 794 | if (IS_ERR(aead)) |
| 795 | return PTR_ERR(aead); |
| 796 | |
| 797 | ctx->child = aead; |
| 798 | |
| 799 | align = crypto_aead_alignmask(aead); |
| 800 | align &= ~(crypto_tfm_ctx_alignment() - 1); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 801 | crypto_aead_set_reqsize( |
| 802 | tfm, |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 803 | sizeof(struct crypto_rfc4106_req_ctx) + |
Herbert Xu | 5d72336 | 2015-05-11 17:47:59 +0800 | [diff] [blame] | 804 | ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 805 | align + 24); |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 806 | |
| 807 | return 0; |
| 808 | } |
| 809 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 810 | static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm) |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 811 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 812 | struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm); |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 813 | |
| 814 | crypto_free_aead(ctx->child); |
| 815 | } |
| 816 | |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 817 | static void crypto_rfc4106_free(struct aead_instance *inst) |
| 818 | { |
| 819 | crypto_drop_aead(aead_instance_ctx(inst)); |
| 820 | kfree(inst); |
| 821 | } |
| 822 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 823 | static int crypto_rfc4106_create(struct crypto_template *tmpl, |
| 824 | struct rtattr **tb) |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 825 | { |
Eric Biggers | cd900f0 | 2020-01-02 19:58:46 -0800 | [diff] [blame] | 826 | u32 mask; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 827 | struct aead_instance *inst; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 828 | struct crypto_aead_spawn *spawn; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 829 | struct aead_alg *alg; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 830 | int err; |
| 831 | |
Eric Biggers | 7bcb2c9 | 2020-07-09 23:20:38 -0700 | [diff] [blame] | 832 | err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); |
| 833 | if (err) |
| 834 | return err; |
Eric Biggers | cd900f0 | 2020-01-02 19:58:46 -0800 | [diff] [blame] | 835 | |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 836 | inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); |
| 837 | if (!inst) |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 838 | return -ENOMEM; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 839 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 840 | spawn = aead_instance_ctx(inst); |
Eric Biggers | cd900f0 | 2020-01-02 19:58:46 -0800 | [diff] [blame] | 841 | err = crypto_grab_aead(spawn, aead_crypto_instance(inst), |
Eric Biggers | 959ac1c | 2020-02-25 20:59:18 -0800 | [diff] [blame] | 842 | crypto_attr_alg_name(tb[1]), 0, mask); |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 843 | if (err) |
Eric Biggers | 959ac1c | 2020-02-25 20:59:18 -0800 | [diff] [blame] | 844 | goto err_free_inst; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 845 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 846 | alg = crypto_spawn_aead_alg(spawn); |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 847 | |
| 848 | err = -EINVAL; |
| 849 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 850 | /* Underlying IV size must be 12. */ |
Corentin LABBE | e0ab7e9 | 2017-08-22 10:08:17 +0200 | [diff] [blame] | 851 | if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE) |
Eric Biggers | 959ac1c | 2020-02-25 20:59:18 -0800 | [diff] [blame] | 852 | goto err_free_inst; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 853 | |
| 854 | /* Not a stream cipher? */ |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 855 | if (alg->base.cra_blocksize != 1) |
Eric Biggers | 959ac1c | 2020-02-25 20:59:18 -0800 | [diff] [blame] | 856 | goto err_free_inst; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 857 | |
| 858 | err = -ENAMETOOLONG; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 859 | if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, |
| 860 | "rfc4106(%s)", alg->base.cra_name) >= |
| 861 | CRYPTO_MAX_ALG_NAME || |
| 862 | snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, |
| 863 | "rfc4106(%s)", alg->base.cra_driver_name) >= |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 864 | CRYPTO_MAX_ALG_NAME) |
Eric Biggers | 959ac1c | 2020-02-25 20:59:18 -0800 | [diff] [blame] | 865 | goto err_free_inst; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 866 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 867 | inst->alg.base.cra_priority = alg->base.cra_priority; |
| 868 | inst->alg.base.cra_blocksize = 1; |
| 869 | inst->alg.base.cra_alignmask = alg->base.cra_alignmask; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 870 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 871 | inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx); |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 872 | |
Corentin LABBE | e0ab7e9 | 2017-08-22 10:08:17 +0200 | [diff] [blame] | 873 | inst->alg.ivsize = GCM_RFC4106_IV_SIZE; |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 874 | inst->alg.chunksize = crypto_aead_alg_chunksize(alg); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 875 | inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 876 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 877 | inst->alg.init = crypto_rfc4106_init_tfm; |
| 878 | inst->alg.exit = crypto_rfc4106_exit_tfm; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 879 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 880 | inst->alg.setkey = crypto_rfc4106_setkey; |
| 881 | inst->alg.setauthsize = crypto_rfc4106_setauthsize; |
| 882 | inst->alg.encrypt = crypto_rfc4106_encrypt; |
| 883 | inst->alg.decrypt = crypto_rfc4106_decrypt; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 884 | |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 885 | inst->free = crypto_rfc4106_free; |
| 886 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 887 | err = aead_register_instance(tmpl, inst); |
Eric Biggers | 959ac1c | 2020-02-25 20:59:18 -0800 | [diff] [blame] | 888 | if (err) { |
| 889 | err_free_inst: |
| 890 | crypto_rfc4106_free(inst); |
| 891 | } |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 892 | return err; |
Herbert Xu | dadbc53 | 2007-12-17 15:33:17 +0800 | [diff] [blame] | 893 | } |
| 894 | |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 895 | static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key, |
| 896 | unsigned int keylen) |
| 897 | { |
| 898 | struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); |
| 899 | struct crypto_aead *child = ctx->child; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 900 | |
| 901 | if (keylen < 4) |
| 902 | return -EINVAL; |
| 903 | |
| 904 | keylen -= 4; |
| 905 | memcpy(ctx->nonce, key + keylen, 4); |
| 906 | |
| 907 | crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); |
| 908 | crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & |
| 909 | CRYPTO_TFM_REQ_MASK); |
Eric Biggers | af5034e | 2019-12-30 21:19:38 -0600 | [diff] [blame] | 910 | return crypto_aead_setkey(child, key, keylen); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 911 | } |
| 912 | |
| 913 | static int crypto_rfc4543_setauthsize(struct crypto_aead *parent, |
| 914 | unsigned int authsize) |
| 915 | { |
| 916 | struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); |
| 917 | |
| 918 | if (authsize != 16) |
| 919 | return -EINVAL; |
| 920 | |
| 921 | return crypto_aead_setauthsize(ctx->child, authsize); |
| 922 | } |
| 923 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 924 | static int crypto_rfc4543_crypt(struct aead_request *req, bool enc) |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 925 | { |
| 926 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
| 927 | struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 928 | struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 929 | struct aead_request *subreq = &rctx->subreq; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 930 | unsigned int authsize = crypto_aead_authsize(aead); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 931 | u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child), |
| 932 | crypto_aead_alignmask(ctx->child) + 1); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 933 | int err; |
| 934 | |
| 935 | if (req->src != req->dst) { |
| 936 | err = crypto_rfc4543_copy_src_to_dst(req, enc); |
| 937 | if (err) |
| 938 | return err; |
| 939 | } |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 940 | |
| 941 | memcpy(iv, ctx->nonce, 4); |
| 942 | memcpy(iv + 4, req->iv, 8); |
| 943 | |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 944 | aead_request_set_tfm(subreq, ctx->child); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 945 | aead_request_set_callback(subreq, req->base.flags, |
| 946 | req->base.complete, req->base.data); |
| 947 | aead_request_set_crypt(subreq, req->src, req->dst, |
| 948 | enc ? 0 : authsize, iv); |
| 949 | aead_request_set_ad(subreq, req->assoclen + req->cryptlen - |
| 950 | subreq->cryptlen); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 951 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 952 | return enc ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 953 | } |
| 954 | |
Jussi Kivilinna | 9489667d | 2013-04-07 16:43:41 +0300 | [diff] [blame] | 955 | static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc) |
| 956 | { |
| 957 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
| 958 | struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); |
| 959 | unsigned int authsize = crypto_aead_authsize(aead); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 960 | unsigned int nbytes = req->assoclen + req->cryptlen - |
| 961 | (enc ? 0 : authsize); |
Kees Cook | 8d60539 | 2018-09-18 19:10:51 -0700 | [diff] [blame] | 962 | SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null); |
Jussi Kivilinna | 9489667d | 2013-04-07 16:43:41 +0300 | [diff] [blame] | 963 | |
Kees Cook | 8d60539 | 2018-09-18 19:10:51 -0700 | [diff] [blame] | 964 | skcipher_request_set_sync_tfm(nreq, ctx->null); |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 965 | skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL); |
| 966 | skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL); |
| 967 | |
| 968 | return crypto_skcipher_encrypt(nreq); |
Jussi Kivilinna | 9489667d | 2013-04-07 16:43:41 +0300 | [diff] [blame] | 969 | } |
| 970 | |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 971 | static int crypto_rfc4543_encrypt(struct aead_request *req) |
| 972 | { |
Iuliana Prodan | 74bf81d | 2019-08-02 11:47:33 +0300 | [diff] [blame] | 973 | return crypto_ipsec_check_assoclen(req->assoclen) ?: |
| 974 | crypto_rfc4543_crypt(req, true); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 975 | } |
| 976 | |
| 977 | static int crypto_rfc4543_decrypt(struct aead_request *req) |
| 978 | { |
Iuliana Prodan | 74bf81d | 2019-08-02 11:47:33 +0300 | [diff] [blame] | 979 | return crypto_ipsec_check_assoclen(req->assoclen) ?: |
| 980 | crypto_rfc4543_crypt(req, false); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 981 | } |
| 982 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 983 | static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm) |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 984 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 985 | struct aead_instance *inst = aead_alg_instance(tfm); |
| 986 | struct crypto_rfc4543_instance_ctx *ictx = aead_instance_ctx(inst); |
Jussi Kivilinna | 9489667d | 2013-04-07 16:43:41 +0300 | [diff] [blame] | 987 | struct crypto_aead_spawn *spawn = &ictx->aead; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 988 | struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 989 | struct crypto_aead *aead; |
Kees Cook | 8d60539 | 2018-09-18 19:10:51 -0700 | [diff] [blame] | 990 | struct crypto_sync_skcipher *null; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 991 | unsigned long align; |
Jussi Kivilinna | 9489667d | 2013-04-07 16:43:41 +0300 | [diff] [blame] | 992 | int err = 0; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 993 | |
| 994 | aead = crypto_spawn_aead(spawn); |
| 995 | if (IS_ERR(aead)) |
| 996 | return PTR_ERR(aead); |
| 997 | |
Eric Biggers | 3a2d4fb | 2017-12-07 10:56:34 -0800 | [diff] [blame] | 998 | null = crypto_get_default_null_skcipher(); |
Jussi Kivilinna | 9489667d | 2013-04-07 16:43:41 +0300 | [diff] [blame] | 999 | err = PTR_ERR(null); |
| 1000 | if (IS_ERR(null)) |
| 1001 | goto err_free_aead; |
| 1002 | |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1003 | ctx->child = aead; |
Jussi Kivilinna | 9489667d | 2013-04-07 16:43:41 +0300 | [diff] [blame] | 1004 | ctx->null = null; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1005 | |
| 1006 | align = crypto_aead_alignmask(aead); |
| 1007 | align &= ~(crypto_tfm_ctx_alignment() - 1); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1008 | crypto_aead_set_reqsize( |
| 1009 | tfm, |
Herbert Xu | 5d72336 | 2015-05-11 17:47:59 +0800 | [diff] [blame] | 1010 | sizeof(struct crypto_rfc4543_req_ctx) + |
| 1011 | ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + |
Corentin LABBE | e0ab7e9 | 2017-08-22 10:08:17 +0200 | [diff] [blame] | 1012 | align + GCM_AES_IV_SIZE); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1013 | |
| 1014 | return 0; |
Jussi Kivilinna | 9489667d | 2013-04-07 16:43:41 +0300 | [diff] [blame] | 1015 | |
| 1016 | err_free_aead: |
| 1017 | crypto_free_aead(aead); |
| 1018 | return err; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1019 | } |
| 1020 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1021 | static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm) |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1022 | { |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1023 | struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1024 | |
| 1025 | crypto_free_aead(ctx->child); |
Eric Biggers | 3a2d4fb | 2017-12-07 10:56:34 -0800 | [diff] [blame] | 1026 | crypto_put_default_null_skcipher(); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1027 | } |
| 1028 | |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 1029 | static void crypto_rfc4543_free(struct aead_instance *inst) |
| 1030 | { |
| 1031 | struct crypto_rfc4543_instance_ctx *ctx = aead_instance_ctx(inst); |
| 1032 | |
| 1033 | crypto_drop_aead(&ctx->aead); |
| 1034 | |
| 1035 | kfree(inst); |
| 1036 | } |
| 1037 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1038 | static int crypto_rfc4543_create(struct crypto_template *tmpl, |
| 1039 | struct rtattr **tb) |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1040 | { |
Eric Biggers | cd900f0 | 2020-01-02 19:58:46 -0800 | [diff] [blame] | 1041 | u32 mask; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1042 | struct aead_instance *inst; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1043 | struct aead_alg *alg; |
Jussi Kivilinna | 9489667d | 2013-04-07 16:43:41 +0300 | [diff] [blame] | 1044 | struct crypto_rfc4543_instance_ctx *ctx; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1045 | int err; |
| 1046 | |
Eric Biggers | 7bcb2c9 | 2020-07-09 23:20:38 -0700 | [diff] [blame] | 1047 | err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); |
| 1048 | if (err) |
| 1049 | return err; |
Eric Biggers | cd900f0 | 2020-01-02 19:58:46 -0800 | [diff] [blame] | 1050 | |
Jussi Kivilinna | 9489667d | 2013-04-07 16:43:41 +0300 | [diff] [blame] | 1051 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1052 | if (!inst) |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1053 | return -ENOMEM; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1054 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1055 | ctx = aead_instance_ctx(inst); |
Eric Biggers | c4caa56 | 2020-02-25 20:59:19 -0800 | [diff] [blame] | 1056 | err = crypto_grab_aead(&ctx->aead, aead_crypto_instance(inst), |
| 1057 | crypto_attr_alg_name(tb[1]), 0, mask); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1058 | if (err) |
Eric Biggers | c4caa56 | 2020-02-25 20:59:19 -0800 | [diff] [blame] | 1059 | goto err_free_inst; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1060 | |
Eric Biggers | c4caa56 | 2020-02-25 20:59:19 -0800 | [diff] [blame] | 1061 | alg = crypto_spawn_aead_alg(&ctx->aead); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1062 | |
| 1063 | err = -EINVAL; |
| 1064 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1065 | /* Underlying IV size must be 12. */ |
Corentin LABBE | e0ab7e9 | 2017-08-22 10:08:17 +0200 | [diff] [blame] | 1066 | if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE) |
Eric Biggers | c4caa56 | 2020-02-25 20:59:19 -0800 | [diff] [blame] | 1067 | goto err_free_inst; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1068 | |
| 1069 | /* Not a stream cipher? */ |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1070 | if (alg->base.cra_blocksize != 1) |
Eric Biggers | c4caa56 | 2020-02-25 20:59:19 -0800 | [diff] [blame] | 1071 | goto err_free_inst; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1072 | |
| 1073 | err = -ENAMETOOLONG; |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1074 | if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, |
| 1075 | "rfc4543(%s)", alg->base.cra_name) >= |
| 1076 | CRYPTO_MAX_ALG_NAME || |
| 1077 | snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, |
| 1078 | "rfc4543(%s)", alg->base.cra_driver_name) >= |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1079 | CRYPTO_MAX_ALG_NAME) |
Eric Biggers | c4caa56 | 2020-02-25 20:59:19 -0800 | [diff] [blame] | 1080 | goto err_free_inst; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1081 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1082 | inst->alg.base.cra_priority = alg->base.cra_priority; |
| 1083 | inst->alg.base.cra_blocksize = 1; |
| 1084 | inst->alg.base.cra_alignmask = alg->base.cra_alignmask; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1085 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1086 | inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1087 | |
Corentin LABBE | e0ab7e9 | 2017-08-22 10:08:17 +0200 | [diff] [blame] | 1088 | inst->alg.ivsize = GCM_RFC4543_IV_SIZE; |
Herbert Xu | 16f37ecd | 2016-07-12 13:17:39 +0800 | [diff] [blame] | 1089 | inst->alg.chunksize = crypto_aead_alg_chunksize(alg); |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1090 | inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1091 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1092 | inst->alg.init = crypto_rfc4543_init_tfm; |
| 1093 | inst->alg.exit = crypto_rfc4543_exit_tfm; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1094 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1095 | inst->alg.setkey = crypto_rfc4543_setkey; |
| 1096 | inst->alg.setauthsize = crypto_rfc4543_setauthsize; |
| 1097 | inst->alg.encrypt = crypto_rfc4543_encrypt; |
| 1098 | inst->alg.decrypt = crypto_rfc4543_decrypt; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1099 | |
Eric Biggers | c4caa56 | 2020-02-25 20:59:19 -0800 | [diff] [blame] | 1100 | inst->free = crypto_rfc4543_free; |
Herbert Xu | 7b05a37 | 2015-07-09 07:17:30 +0800 | [diff] [blame] | 1101 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1102 | err = aead_register_instance(tmpl, inst); |
Eric Biggers | c4caa56 | 2020-02-25 20:59:19 -0800 | [diff] [blame] | 1103 | if (err) { |
| 1104 | err_free_inst: |
| 1105 | crypto_rfc4543_free(inst); |
| 1106 | } |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1107 | return err; |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1108 | } |
| 1109 | |
Xiongfeng Wang | 56a00d9 | 2019-01-18 13:58:13 +0800 | [diff] [blame] | 1110 | static struct crypto_template crypto_gcm_tmpls[] = { |
| 1111 | { |
| 1112 | .name = "gcm_base", |
| 1113 | .create = crypto_gcm_base_create, |
| 1114 | .module = THIS_MODULE, |
| 1115 | }, { |
| 1116 | .name = "gcm", |
| 1117 | .create = crypto_gcm_create, |
| 1118 | .module = THIS_MODULE, |
| 1119 | }, { |
| 1120 | .name = "rfc4106", |
| 1121 | .create = crypto_rfc4106_create, |
| 1122 | .module = THIS_MODULE, |
| 1123 | }, { |
| 1124 | .name = "rfc4543", |
| 1125 | .create = crypto_rfc4543_create, |
| 1126 | .module = THIS_MODULE, |
| 1127 | }, |
Tobias Brunner | 73c89c1 | 2010-01-17 21:52:11 +1100 | [diff] [blame] | 1128 | }; |
| 1129 | |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 1130 | static int __init crypto_gcm_module_init(void) |
| 1131 | { |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 1132 | int err; |
| 1133 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1134 | gcm_zeroes = kzalloc(sizeof(*gcm_zeroes), GFP_KERNEL); |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 1135 | if (!gcm_zeroes) |
| 1136 | return -ENOMEM; |
| 1137 | |
Herbert Xu | adcbc68 | 2015-06-16 13:54:18 +0800 | [diff] [blame] | 1138 | sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf)); |
| 1139 | |
Xiongfeng Wang | 56a00d9 | 2019-01-18 13:58:13 +0800 | [diff] [blame] | 1140 | err = crypto_register_templates(crypto_gcm_tmpls, |
| 1141 | ARRAY_SIZE(crypto_gcm_tmpls)); |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 1142 | if (err) |
Xiongfeng Wang | 56a00d9 | 2019-01-18 13:58:13 +0800 | [diff] [blame] | 1143 | kfree(gcm_zeroes); |
Herbert Xu | d00aa19 | 2007-12-07 20:31:10 +0800 | [diff] [blame] | 1144 | |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 1145 | return err; |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 1146 | } |
| 1147 | |
| 1148 | static void __exit crypto_gcm_module_exit(void) |
| 1149 | { |
Huang Ying | 9382d97 | 2009-08-06 15:34:26 +1000 | [diff] [blame] | 1150 | kfree(gcm_zeroes); |
Xiongfeng Wang | 56a00d9 | 2019-01-18 13:58:13 +0800 | [diff] [blame] | 1151 | crypto_unregister_templates(crypto_gcm_tmpls, |
| 1152 | ARRAY_SIZE(crypto_gcm_tmpls)); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 1153 | } |
| 1154 | |
Eric Biggers | c4741b2 | 2019-04-11 21:57:42 -0700 | [diff] [blame] | 1155 | subsys_initcall(crypto_gcm_module_init); |
Mikko Herranen | 28db8e3 | 2007-11-26 22:24:11 +0800 | [diff] [blame] | 1156 | module_exit(crypto_gcm_module_exit); |
| 1157 | |
| 1158 | MODULE_LICENSE("GPL"); |
| 1159 | MODULE_DESCRIPTION("Galois/Counter Mode"); |
| 1160 | MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>"); |
Kees Cook | 5d26a10 | 2014-11-20 17:05:53 -0800 | [diff] [blame] | 1161 | MODULE_ALIAS_CRYPTO("gcm_base"); |
| 1162 | MODULE_ALIAS_CRYPTO("rfc4106"); |
| 1163 | MODULE_ALIAS_CRYPTO("rfc4543"); |
Kees Cook | 4943ba1 | 2014-11-24 16:32:38 -0800 | [diff] [blame] | 1164 | MODULE_ALIAS_CRYPTO("gcm"); |