blob: 371e14f0a20392d9e43ed45084f6a8f04dc87949 [file] [log] [blame]
Hannes Reineckef50fff72022-06-27 11:52:02 +02001// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2020 Hannes Reinecke, SUSE Linux
4 */
5
6#include <linux/crc32.h>
7#include <linux/base64.h>
8#include <linux/prandom.h>
9#include <asm/unaligned.h>
10#include <crypto/hash.h>
11#include <crypto/dh.h>
12#include "nvme.h"
13#include "fabrics.h"
14#include <linux/nvme-auth.h>
15
Sagi Grimbergb7d604c2022-11-13 13:24:13 +020016#define CHAP_BUF_SIZE 4096
Sagi Grimberge481fc02022-11-15 17:08:06 +010017static struct kmem_cache *nvme_chap_buf_cache;
18static mempool_t *nvme_chap_buf_pool;
Sagi Grimbergb7d604c2022-11-13 13:24:13 +020019
Hannes Reineckef50fff72022-06-27 11:52:02 +020020struct nvme_dhchap_queue_context {
21 struct list_head entry;
22 struct work_struct auth_work;
23 struct nvme_ctrl *ctrl;
24 struct crypto_shash *shash_tfm;
Hannes Reineckeb61775d2022-06-27 11:52:03 +020025 struct crypto_kpp *dh_tfm;
Mark O'Donovanf047dae2023-10-17 17:09:18 +000026 struct nvme_dhchap_key *transformed_key;
Hannes Reineckef50fff72022-06-27 11:52:02 +020027 void *buf;
Hannes Reineckef50fff72022-06-27 11:52:02 +020028 int qid;
29 int error;
30 u32 s1;
31 u32 s2;
Mark O'Donovanfc1e03e2023-10-25 10:51:24 +000032 bool bi_directional;
Hannes Reineckef50fff72022-06-27 11:52:02 +020033 u16 transaction;
34 u8 status;
Christophe JAILLET0f5335e2023-05-01 14:40:28 +020035 u8 dhgroup_id;
Hannes Reineckef50fff72022-06-27 11:52:02 +020036 u8 hash_id;
37 size_t hash_len;
Hannes Reineckef50fff72022-06-27 11:52:02 +020038 u8 c1[64];
39 u8 c2[64];
40 u8 response[64];
Hannes Reineckeb61775d2022-06-27 11:52:03 +020041 u8 *ctrl_key;
Hannes Reineckeb61775d2022-06-27 11:52:03 +020042 u8 *host_key;
Hannes Reineckeb61775d2022-06-27 11:52:03 +020043 u8 *sess_key;
Christophe JAILLET0f5335e2023-05-01 14:40:28 +020044 int ctrl_key_len;
45 int host_key_len;
Hannes Reineckeb61775d2022-06-27 11:52:03 +020046 int sess_key_len;
Hannes Reineckef50fff72022-06-27 11:52:02 +020047};
48
Tom Rix70daa5c2023-02-06 06:57:00 -080049static struct workqueue_struct *nvme_auth_wq;
Shin'ichiro Kawasakibd97a592023-01-31 18:26:44 +090050
Sagi Grimbergaa36d712022-11-13 13:24:20 +020051static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl)
52{
53 return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues +
54 ctrl->opts->nr_poll_queues + 1;
55}
56
Hannes Reineckef50fff72022-06-27 11:52:02 +020057static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid,
58 void *data, size_t data_len, bool auth_send)
59{
60 struct nvme_command cmd = {};
Hannes Reinecke48dae462024-01-29 07:39:48 +010061 nvme_submit_flags_t flags = NVME_SUBMIT_RETRY;
Hannes Reineckef9ddefb2024-01-29 07:39:45 +010062 struct request_queue *q = ctrl->fabrics_q;
Hannes Reineckef50fff72022-06-27 11:52:02 +020063 int ret;
64
Hannes Reineckef9ddefb2024-01-29 07:39:45 +010065 if (qid != 0) {
Hannes Reineckebd2687f2024-01-29 07:39:46 +010066 flags |= NVME_SUBMIT_NOWAIT | NVME_SUBMIT_RESERVED;
Hannes Reineckef9ddefb2024-01-29 07:39:45 +010067 q = ctrl->connect_q;
68 }
69
Hannes Reineckef50fff72022-06-27 11:52:02 +020070 cmd.auth_common.opcode = nvme_fabrics_command;
71 cmd.auth_common.secp = NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER;
72 cmd.auth_common.spsp0 = 0x01;
73 cmd.auth_common.spsp1 = 0x01;
74 if (auth_send) {
75 cmd.auth_send.fctype = nvme_fabrics_type_auth_send;
76 cmd.auth_send.tl = cpu_to_le32(data_len);
77 } else {
78 cmd.auth_receive.fctype = nvme_fabrics_type_auth_receive;
79 cmd.auth_receive.al = cpu_to_le32(data_len);
80 }
81
82 ret = __nvme_submit_sync_cmd(q, &cmd, NULL, data, data_len,
Hannes Reineckebd2687f2024-01-29 07:39:46 +010083 qid == 0 ? NVME_QID_ANY : qid, flags);
Hannes Reineckef50fff72022-06-27 11:52:02 +020084 if (ret > 0)
85 dev_warn(ctrl->device,
86 "qid %d auth_send failed with status %d\n", qid, ret);
87 else if (ret < 0)
88 dev_err(ctrl->device,
89 "qid %d auth_send failed with error %d\n", qid, ret);
90 return ret;
91}
92
93static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid,
94 struct nvmf_auth_dhchap_failure_data *data,
95 u16 transaction, u8 expected_msg)
96{
97 dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n",
98 __func__, qid, data->auth_type, data->auth_id);
99
100 if (data->auth_type == NVME_AUTH_COMMON_MESSAGES &&
101 data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
102 return data->rescode_exp;
103 }
104 if (data->auth_type != NVME_AUTH_DHCHAP_MESSAGES ||
105 data->auth_id != expected_msg) {
106 dev_warn(ctrl->device,
107 "qid %d invalid message %02x/%02x\n",
108 qid, data->auth_type, data->auth_id);
109 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
110 }
111 if (le16_to_cpu(data->t_id) != transaction) {
112 dev_warn(ctrl->device,
113 "qid %d invalid transaction ID %d\n",
114 qid, le16_to_cpu(data->t_id));
115 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
116 }
117 return 0;
118}
119
120static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl,
121 struct nvme_dhchap_queue_context *chap)
122{
123 struct nvmf_auth_dhchap_negotiate_data *data = chap->buf;
124 size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol);
125
Sagi Grimbergb7d604c2022-11-13 13:24:13 +0200126 if (size > CHAP_BUF_SIZE) {
Hannes Reineckef50fff72022-06-27 11:52:02 +0200127 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
128 return -EINVAL;
129 }
130 memset((u8 *)chap->buf, 0, size);
131 data->auth_type = NVME_AUTH_COMMON_MESSAGES;
132 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
133 data->t_id = cpu_to_le16(chap->transaction);
134 data->sc_c = 0; /* No secure channel concatenation */
135 data->napd = 1;
136 data->auth_protocol[0].dhchap.authid = NVME_AUTH_DHCHAP_AUTH_ID;
137 data->auth_protocol[0].dhchap.halen = 3;
138 data->auth_protocol[0].dhchap.dhlen = 6;
139 data->auth_protocol[0].dhchap.idlist[0] = NVME_AUTH_HASH_SHA256;
140 data->auth_protocol[0].dhchap.idlist[1] = NVME_AUTH_HASH_SHA384;
141 data->auth_protocol[0].dhchap.idlist[2] = NVME_AUTH_HASH_SHA512;
142 data->auth_protocol[0].dhchap.idlist[30] = NVME_AUTH_DHGROUP_NULL;
143 data->auth_protocol[0].dhchap.idlist[31] = NVME_AUTH_DHGROUP_2048;
144 data->auth_protocol[0].dhchap.idlist[32] = NVME_AUTH_DHGROUP_3072;
145 data->auth_protocol[0].dhchap.idlist[33] = NVME_AUTH_DHGROUP_4096;
146 data->auth_protocol[0].dhchap.idlist[34] = NVME_AUTH_DHGROUP_6144;
147 data->auth_protocol[0].dhchap.idlist[35] = NVME_AUTH_DHGROUP_8192;
148
149 return size;
150}
151
152static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl,
153 struct nvme_dhchap_queue_context *chap)
154{
155 struct nvmf_auth_dhchap_challenge_data *data = chap->buf;
156 u16 dhvlen = le16_to_cpu(data->dhvlen);
157 size_t size = sizeof(*data) + data->hl + dhvlen;
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200158 const char *gid_name = nvme_auth_dhgroup_name(data->dhgid);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200159 const char *hmac_name, *kpp_name;
160
Sagi Grimbergb7d604c2022-11-13 13:24:13 +0200161 if (size > CHAP_BUF_SIZE) {
Hannes Reineckef50fff72022-06-27 11:52:02 +0200162 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
Hannes Reineckeb0ef1b12022-12-13 20:00:26 +0100163 return -EINVAL;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200164 }
165
166 hmac_name = nvme_auth_hmac_name(data->hashid);
167 if (!hmac_name) {
168 dev_warn(ctrl->device,
169 "qid %d: invalid HASH ID %d\n",
170 chap->qid, data->hashid);
171 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
Hannes Reineckeb0ef1b12022-12-13 20:00:26 +0100172 return -EPROTO;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200173 }
174
175 if (chap->hash_id == data->hashid && chap->shash_tfm &&
176 !strcmp(crypto_shash_alg_name(chap->shash_tfm), hmac_name) &&
177 crypto_shash_digestsize(chap->shash_tfm) == data->hl) {
178 dev_dbg(ctrl->device,
179 "qid %d: reuse existing hash %s\n",
180 chap->qid, hmac_name);
181 goto select_kpp;
182 }
183
184 /* Reset if hash cannot be reused */
185 if (chap->shash_tfm) {
186 crypto_free_shash(chap->shash_tfm);
187 chap->hash_id = 0;
188 chap->hash_len = 0;
189 }
190 chap->shash_tfm = crypto_alloc_shash(hmac_name, 0,
191 CRYPTO_ALG_ALLOCATES_MEMORY);
192 if (IS_ERR(chap->shash_tfm)) {
193 dev_warn(ctrl->device,
194 "qid %d: failed to allocate hash %s, error %ld\n",
195 chap->qid, hmac_name, PTR_ERR(chap->shash_tfm));
196 chap->shash_tfm = NULL;
197 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
Hannes Reineckeb0ef1b12022-12-13 20:00:26 +0100198 return -ENOMEM;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200199 }
200
201 if (crypto_shash_digestsize(chap->shash_tfm) != data->hl) {
202 dev_warn(ctrl->device,
203 "qid %d: invalid hash length %d\n",
204 chap->qid, data->hl);
205 crypto_free_shash(chap->shash_tfm);
206 chap->shash_tfm = NULL;
207 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
Hannes Reineckeb0ef1b12022-12-13 20:00:26 +0100208 return -EPROTO;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200209 }
210
Hannes Reineckef50fff72022-06-27 11:52:02 +0200211 chap->hash_id = data->hashid;
212 chap->hash_len = data->hl;
213 dev_dbg(ctrl->device, "qid %d: selected hash %s\n",
214 chap->qid, hmac_name);
215
216select_kpp:
217 kpp_name = nvme_auth_dhgroup_kpp(data->dhgid);
218 if (!kpp_name) {
219 dev_warn(ctrl->device,
220 "qid %d: invalid DH group id %d\n",
221 chap->qid, data->dhgid);
222 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200223 /* Leave previous dh_tfm intact */
Hannes Reineckeb0ef1b12022-12-13 20:00:26 +0100224 return -EPROTO;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200225 }
226
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200227 if (chap->dhgroup_id == data->dhgid &&
228 (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) {
229 dev_dbg(ctrl->device,
230 "qid %d: reuse existing DH group %s\n",
231 chap->qid, gid_name);
232 goto skip_kpp;
233 }
234
235 /* Reset dh_tfm if it can't be reused */
236 if (chap->dh_tfm) {
237 crypto_free_kpp(chap->dh_tfm);
238 chap->dh_tfm = NULL;
239 }
240
Hannes Reineckef50fff72022-06-27 11:52:02 +0200241 if (data->dhgid != NVME_AUTH_DHGROUP_NULL) {
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200242 if (dhvlen == 0) {
243 dev_warn(ctrl->device,
244 "qid %d: empty DH value\n",
245 chap->qid);
246 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
Hannes Reineckeb0ef1b12022-12-13 20:00:26 +0100247 return -EPROTO;
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200248 }
249
250 chap->dh_tfm = crypto_alloc_kpp(kpp_name, 0, 0);
251 if (IS_ERR(chap->dh_tfm)) {
252 int ret = PTR_ERR(chap->dh_tfm);
253
254 dev_warn(ctrl->device,
255 "qid %d: error %d initializing DH group %s\n",
256 chap->qid, ret, gid_name);
257 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
258 chap->dh_tfm = NULL;
Dan Carpenter51d24f72023-02-16 15:14:49 +0300259 return ret;
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200260 }
261 dev_dbg(ctrl->device, "qid %d: selected DH group %s\n",
262 chap->qid, gid_name);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200263 } else if (dhvlen != 0) {
264 dev_warn(ctrl->device,
265 "qid %d: invalid DH value for NULL DH\n",
266 chap->qid);
267 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
Hannes Reineckeb0ef1b12022-12-13 20:00:26 +0100268 return -EPROTO;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200269 }
270 chap->dhgroup_id = data->dhgid;
271
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200272skip_kpp:
Hannes Reineckef50fff72022-06-27 11:52:02 +0200273 chap->s1 = le32_to_cpu(data->seqnum);
274 memcpy(chap->c1, data->cval, chap->hash_len);
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200275 if (dhvlen) {
276 chap->ctrl_key = kmalloc(dhvlen, GFP_KERNEL);
277 if (!chap->ctrl_key) {
278 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
Hannes Reineckeb0ef1b12022-12-13 20:00:26 +0100279 return -ENOMEM;
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200280 }
281 chap->ctrl_key_len = dhvlen;
282 memcpy(chap->ctrl_key, data->cval + chap->hash_len,
283 dhvlen);
284 dev_dbg(ctrl->device, "ctrl public key %*ph\n",
285 (int)chap->ctrl_key_len, chap->ctrl_key);
286 }
Hannes Reineckef50fff72022-06-27 11:52:02 +0200287
288 return 0;
289}
290
291static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl,
292 struct nvme_dhchap_queue_context *chap)
293{
294 struct nvmf_auth_dhchap_reply_data *data = chap->buf;
295 size_t size = sizeof(*data);
296
297 size += 2 * chap->hash_len;
298
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200299 if (chap->host_key_len)
300 size += chap->host_key_len;
301
Sagi Grimbergb7d604c2022-11-13 13:24:13 +0200302 if (size > CHAP_BUF_SIZE) {
Hannes Reineckef50fff72022-06-27 11:52:02 +0200303 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
304 return -EINVAL;
305 }
306
307 memset(chap->buf, 0, size);
308 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
309 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_REPLY;
310 data->t_id = cpu_to_le16(chap->transaction);
311 data->hl = chap->hash_len;
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200312 data->dhvlen = cpu_to_le16(chap->host_key_len);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200313 memcpy(data->rval, chap->response, chap->hash_len);
314 if (ctrl->ctrl_key) {
Mark O'Donovanfc1e03e2023-10-25 10:51:24 +0000315 chap->bi_directional = true;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200316 get_random_bytes(chap->c2, chap->hash_len);
317 data->cvalid = 1;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200318 memcpy(data->rval + chap->hash_len, chap->c2,
319 chap->hash_len);
320 dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n",
321 __func__, chap->qid, (int)chap->hash_len, chap->c2);
322 } else {
323 memset(chap->c2, 0, chap->hash_len);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200324 }
Mark O'Donovan6f66d042023-10-25 10:51:25 +0000325 chap->s2 = nvme_auth_get_seqnum();
Hannes Reineckef50fff72022-06-27 11:52:02 +0200326 data->seqnum = cpu_to_le32(chap->s2);
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200327 if (chap->host_key_len) {
328 dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n",
329 __func__, chap->qid,
330 chap->host_key_len, chap->host_key);
331 memcpy(data->rval + 2 * chap->hash_len, chap->host_key,
332 chap->host_key_len);
333 }
334
Hannes Reineckef50fff72022-06-27 11:52:02 +0200335 return size;
336}
337
338static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl,
339 struct nvme_dhchap_queue_context *chap)
340{
341 struct nvmf_auth_dhchap_success1_data *data = chap->buf;
Mark O'Donovan75276842023-10-25 10:51:23 +0000342 size_t size = sizeof(*data) + chap->hash_len;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200343
Sagi Grimbergb7d604c2022-11-13 13:24:13 +0200344 if (size > CHAP_BUF_SIZE) {
Hannes Reineckef50fff72022-06-27 11:52:02 +0200345 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
Hannes Reineckeb0ef1b12022-12-13 20:00:26 +0100346 return -EINVAL;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200347 }
348
349 if (data->hl != chap->hash_len) {
350 dev_warn(ctrl->device,
351 "qid %d: invalid hash length %u\n",
352 chap->qid, data->hl);
353 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
Hannes Reineckeb0ef1b12022-12-13 20:00:26 +0100354 return -EPROTO;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200355 }
356
357 /* Just print out information for the admin queue */
358 if (chap->qid == 0)
359 dev_info(ctrl->device,
360 "qid 0: authenticated with hash %s dhgroup %s\n",
361 nvme_auth_hmac_name(chap->hash_id),
362 nvme_auth_dhgroup_name(chap->dhgroup_id));
363
364 if (!data->rvalid)
365 return 0;
366
367 /* Validate controller response */
368 if (memcmp(chap->response, data->rval, data->hl)) {
369 dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n",
370 __func__, chap->qid, (int)chap->hash_len, data->rval);
371 dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n",
372 __func__, chap->qid, (int)chap->hash_len,
373 chap->response);
374 dev_warn(ctrl->device,
375 "qid %d: controller authentication failed\n",
376 chap->qid);
377 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
Hannes Reineckeb0ef1b12022-12-13 20:00:26 +0100378 return -ECONNREFUSED;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200379 }
380
381 /* Just print out information for the admin queue */
382 if (chap->qid == 0)
383 dev_info(ctrl->device,
384 "qid 0: controller authenticated\n");
385 return 0;
386}
387
388static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl,
389 struct nvme_dhchap_queue_context *chap)
390{
391 struct nvmf_auth_dhchap_success2_data *data = chap->buf;
392 size_t size = sizeof(*data);
393
394 memset(chap->buf, 0, size);
395 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
396 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2;
397 data->t_id = cpu_to_le16(chap->transaction);
398
399 return size;
400}
401
402static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl,
403 struct nvme_dhchap_queue_context *chap)
404{
405 struct nvmf_auth_dhchap_failure_data *data = chap->buf;
406 size_t size = sizeof(*data);
407
408 memset(chap->buf, 0, size);
409 data->auth_type = NVME_AUTH_COMMON_MESSAGES;
410 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
411 data->t_id = cpu_to_le16(chap->transaction);
412 data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED;
413 data->rescode_exp = chap->status;
414
415 return size;
416}
417
418static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl,
419 struct nvme_dhchap_queue_context *chap)
420{
421 SHASH_DESC_ON_STACK(shash, chap->shash_tfm);
422 u8 buf[4], *challenge = chap->c1;
423 int ret;
424
425 dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n",
426 __func__, chap->qid, chap->s1, chap->transaction);
427
Mark O'Donovanf047dae2023-10-17 17:09:18 +0000428 if (!chap->transformed_key) {
429 chap->transformed_key = nvme_auth_transform_key(ctrl->host_key,
Hannes Reineckef50fff72022-06-27 11:52:02 +0200430 ctrl->opts->host->nqn);
Mark O'Donovanf047dae2023-10-17 17:09:18 +0000431 if (IS_ERR(chap->transformed_key)) {
432 ret = PTR_ERR(chap->transformed_key);
433 chap->transformed_key = NULL;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200434 return ret;
435 }
436 } else {
437 dev_dbg(ctrl->device, "%s: qid %d re-using host response\n",
438 __func__, chap->qid);
439 }
440
441 ret = crypto_shash_setkey(chap->shash_tfm,
Mark O'Donovanf047dae2023-10-17 17:09:18 +0000442 chap->transformed_key->key, chap->transformed_key->len);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200443 if (ret) {
444 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",
445 chap->qid, ret);
446 goto out;
447 }
448
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200449 if (chap->dh_tfm) {
450 challenge = kmalloc(chap->hash_len, GFP_KERNEL);
451 if (!challenge) {
452 ret = -ENOMEM;
453 goto out;
454 }
455 ret = nvme_auth_augmented_challenge(chap->hash_id,
456 chap->sess_key,
457 chap->sess_key_len,
458 chap->c1, challenge,
459 chap->hash_len);
460 if (ret)
461 goto out;
462 }
463
Hannes Reineckef50fff72022-06-27 11:52:02 +0200464 shash->tfm = chap->shash_tfm;
465 ret = crypto_shash_init(shash);
466 if (ret)
467 goto out;
468 ret = crypto_shash_update(shash, challenge, chap->hash_len);
469 if (ret)
470 goto out;
471 put_unaligned_le32(chap->s1, buf);
472 ret = crypto_shash_update(shash, buf, 4);
473 if (ret)
474 goto out;
475 put_unaligned_le16(chap->transaction, buf);
476 ret = crypto_shash_update(shash, buf, 2);
477 if (ret)
478 goto out;
479 memset(buf, 0, sizeof(buf));
480 ret = crypto_shash_update(shash, buf, 1);
481 if (ret)
482 goto out;
483 ret = crypto_shash_update(shash, "HostHost", 8);
484 if (ret)
485 goto out;
486 ret = crypto_shash_update(shash, ctrl->opts->host->nqn,
487 strlen(ctrl->opts->host->nqn));
488 if (ret)
489 goto out;
490 ret = crypto_shash_update(shash, buf, 1);
491 if (ret)
492 goto out;
493 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,
494 strlen(ctrl->opts->subsysnqn));
495 if (ret)
496 goto out;
497 ret = crypto_shash_final(shash, chap->response);
498out:
499 if (challenge != chap->c1)
500 kfree(challenge);
501 return ret;
502}
503
504static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl,
505 struct nvme_dhchap_queue_context *chap)
506{
507 SHASH_DESC_ON_STACK(shash, chap->shash_tfm);
Mark O'Donovanf047dae2023-10-17 17:09:18 +0000508 struct nvme_dhchap_key *transformed_key;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200509 u8 buf[4], *challenge = chap->c2;
510 int ret;
511
Mark O'Donovanf047dae2023-10-17 17:09:18 +0000512 transformed_key = nvme_auth_transform_key(ctrl->ctrl_key,
Hannes Reineckef50fff72022-06-27 11:52:02 +0200513 ctrl->opts->subsysnqn);
Mark O'Donovanf047dae2023-10-17 17:09:18 +0000514 if (IS_ERR(transformed_key)) {
515 ret = PTR_ERR(transformed_key);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200516 return ret;
517 }
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200518
Hannes Reineckef50fff72022-06-27 11:52:02 +0200519 ret = crypto_shash_setkey(chap->shash_tfm,
Mark O'Donovanf047dae2023-10-17 17:09:18 +0000520 transformed_key->key, transformed_key->len);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200521 if (ret) {
522 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n",
523 chap->qid, ret);
524 goto out;
525 }
526
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200527 if (chap->dh_tfm) {
528 challenge = kmalloc(chap->hash_len, GFP_KERNEL);
529 if (!challenge) {
530 ret = -ENOMEM;
531 goto out;
532 }
533 ret = nvme_auth_augmented_challenge(chap->hash_id,
534 chap->sess_key,
535 chap->sess_key_len,
536 chap->c2, challenge,
537 chap->hash_len);
538 if (ret)
539 goto out;
540 }
Hannes Reineckef50fff72022-06-27 11:52:02 +0200541 dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n",
542 __func__, chap->qid, chap->s2, chap->transaction);
543 dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n",
544 __func__, chap->qid, (int)chap->hash_len, challenge);
545 dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n",
546 __func__, chap->qid, ctrl->opts->subsysnqn);
547 dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n",
548 __func__, chap->qid, ctrl->opts->host->nqn);
549 shash->tfm = chap->shash_tfm;
550 ret = crypto_shash_init(shash);
551 if (ret)
552 goto out;
553 ret = crypto_shash_update(shash, challenge, chap->hash_len);
554 if (ret)
555 goto out;
556 put_unaligned_le32(chap->s2, buf);
557 ret = crypto_shash_update(shash, buf, 4);
558 if (ret)
559 goto out;
560 put_unaligned_le16(chap->transaction, buf);
561 ret = crypto_shash_update(shash, buf, 2);
562 if (ret)
563 goto out;
564 memset(buf, 0, 4);
565 ret = crypto_shash_update(shash, buf, 1);
566 if (ret)
567 goto out;
568 ret = crypto_shash_update(shash, "Controller", 10);
569 if (ret)
570 goto out;
571 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn,
572 strlen(ctrl->opts->subsysnqn));
573 if (ret)
574 goto out;
575 ret = crypto_shash_update(shash, buf, 1);
576 if (ret)
577 goto out;
578 ret = crypto_shash_update(shash, ctrl->opts->host->nqn,
579 strlen(ctrl->opts->host->nqn));
580 if (ret)
581 goto out;
582 ret = crypto_shash_final(shash, chap->response);
583out:
584 if (challenge != chap->c2)
585 kfree(challenge);
Mark O'Donovanf047dae2023-10-17 17:09:18 +0000586 nvme_auth_free_key(transformed_key);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200587 return ret;
588}
589
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200590static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl,
591 struct nvme_dhchap_queue_context *chap)
592{
593 int ret;
594
595 if (chap->host_key && chap->host_key_len) {
596 dev_dbg(ctrl->device,
597 "qid %d: reusing host key\n", chap->qid);
598 goto gen_sesskey;
599 }
600 ret = nvme_auth_gen_privkey(chap->dh_tfm, chap->dhgroup_id);
601 if (ret < 0) {
602 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
603 return ret;
604 }
605
606 chap->host_key_len = crypto_kpp_maxsize(chap->dh_tfm);
607
608 chap->host_key = kzalloc(chap->host_key_len, GFP_KERNEL);
609 if (!chap->host_key) {
610 chap->host_key_len = 0;
611 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
612 return -ENOMEM;
613 }
614 ret = nvme_auth_gen_pubkey(chap->dh_tfm,
615 chap->host_key, chap->host_key_len);
616 if (ret) {
617 dev_dbg(ctrl->device,
618 "failed to generate public key, error %d\n", ret);
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200619 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
620 return ret;
621 }
622
623gen_sesskey:
624 chap->sess_key_len = chap->host_key_len;
625 chap->sess_key = kmalloc(chap->sess_key_len, GFP_KERNEL);
626 if (!chap->sess_key) {
627 chap->sess_key_len = 0;
628 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
629 return -ENOMEM;
630 }
631
632 ret = nvme_auth_gen_shared_secret(chap->dh_tfm,
633 chap->ctrl_key, chap->ctrl_key_len,
634 chap->sess_key, chap->sess_key_len);
635 if (ret) {
636 dev_dbg(ctrl->device,
637 "failed to generate shared secret, error %d\n", ret);
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200638 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
639 return ret;
640 }
641 dev_dbg(ctrl->device, "shared secret %*ph\n",
642 (int)chap->sess_key_len, chap->sess_key);
643 return 0;
644}
645
Sagi Grimberg0a7ce372022-11-13 13:24:05 +0200646static void nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context *chap)
Hannes Reineckef50fff72022-06-27 11:52:02 +0200647{
Mark O'Donovanf047dae2023-10-17 17:09:18 +0000648 nvme_auth_free_key(chap->transformed_key);
649 chap->transformed_key = NULL;
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200650 kfree_sensitive(chap->host_key);
651 chap->host_key = NULL;
652 chap->host_key_len = 0;
653 kfree_sensitive(chap->ctrl_key);
654 chap->ctrl_key = NULL;
655 chap->ctrl_key_len = 0;
656 kfree_sensitive(chap->sess_key);
657 chap->sess_key = NULL;
658 chap->sess_key_len = 0;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200659 chap->status = 0;
660 chap->error = 0;
661 chap->s1 = 0;
662 chap->s2 = 0;
Mark O'Donovanfc1e03e2023-10-25 10:51:24 +0000663 chap->bi_directional = false;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200664 chap->transaction = 0;
665 memset(chap->c1, 0, sizeof(chap->c1));
666 memset(chap->c2, 0, sizeof(chap->c2));
Sagi Grimberge481fc02022-11-15 17:08:06 +0100667 mempool_free(chap->buf, nvme_chap_buf_pool);
Sagi Grimbergb7d604c2022-11-13 13:24:13 +0200668 chap->buf = NULL;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200669}
670
Sagi Grimberg0a7ce372022-11-13 13:24:05 +0200671static void nvme_auth_free_dhchap(struct nvme_dhchap_queue_context *chap)
Hannes Reineckef50fff72022-06-27 11:52:02 +0200672{
Sagi Grimberg0a7ce372022-11-13 13:24:05 +0200673 nvme_auth_reset_dhchap(chap);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200674 if (chap->shash_tfm)
675 crypto_free_shash(chap->shash_tfm);
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200676 if (chap->dh_tfm)
677 crypto_free_kpp(chap->dh_tfm);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200678}
679
Sagi Grimberg0c999e62022-11-13 13:24:06 +0200680static void nvme_queue_auth_work(struct work_struct *work)
Hannes Reineckef50fff72022-06-27 11:52:02 +0200681{
682 struct nvme_dhchap_queue_context *chap =
683 container_of(work, struct nvme_dhchap_queue_context, auth_work);
684 struct nvme_ctrl *ctrl = chap->ctrl;
685 size_t tl;
686 int ret = 0;
687
Sagi Grimbergb7d604c2022-11-13 13:24:13 +0200688 /*
689 * Allocate a large enough buffer for the entire negotiation:
690 * 4k is enough to ffdhe8192.
691 */
Sagi Grimberge481fc02022-11-15 17:08:06 +0100692 chap->buf = mempool_alloc(nvme_chap_buf_pool, GFP_KERNEL);
Sagi Grimbergb7d604c2022-11-13 13:24:13 +0200693 if (!chap->buf) {
694 chap->error = -ENOMEM;
695 return;
696 }
697
Hannes Reineckef50fff72022-06-27 11:52:02 +0200698 chap->transaction = ctrl->transaction++;
699
700 /* DH-HMAC-CHAP Step 1: send negotiate */
701 dev_dbg(ctrl->device, "%s: qid %d send negotiate\n",
702 __func__, chap->qid);
703 ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap);
704 if (ret < 0) {
705 chap->error = ret;
706 return;
707 }
708 tl = ret;
709 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
710 if (ret) {
711 chap->error = ret;
712 return;
713 }
714
715 /* DH-HMAC-CHAP Step 2: receive challenge */
716 dev_dbg(ctrl->device, "%s: qid %d receive challenge\n",
717 __func__, chap->qid);
718
Sagi Grimbergb7d604c2022-11-13 13:24:13 +0200719 memset(chap->buf, 0, CHAP_BUF_SIZE);
720 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,
721 false);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200722 if (ret) {
723 dev_warn(ctrl->device,
724 "qid %d failed to receive challenge, %s %d\n",
725 chap->qid, ret < 0 ? "error" : "nvme status", ret);
726 chap->error = ret;
727 return;
728 }
729 ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction,
730 NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE);
731 if (ret) {
732 chap->status = ret;
Daniel Wagner0e34bd92024-04-30 15:19:28 +0200733 chap->error = -EKEYREJECTED;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200734 return;
735 }
736
737 ret = nvme_auth_process_dhchap_challenge(ctrl, chap);
738 if (ret) {
739 /* Invalid challenge parameters */
740 chap->error = ret;
741 goto fail2;
742 }
743
Hannes Reineckeb61775d2022-06-27 11:52:03 +0200744 if (chap->ctrl_key_len) {
745 dev_dbg(ctrl->device,
746 "%s: qid %d DH exponential\n",
747 __func__, chap->qid);
748 ret = nvme_auth_dhchap_exponential(ctrl, chap);
749 if (ret) {
750 chap->error = ret;
751 goto fail2;
752 }
753 }
754
Hannes Reineckef50fff72022-06-27 11:52:02 +0200755 dev_dbg(ctrl->device, "%s: qid %d host response\n",
756 __func__, chap->qid);
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200757 mutex_lock(&ctrl->dhchap_auth_mutex);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200758 ret = nvme_auth_dhchap_setup_host_response(ctrl, chap);
Mark O'Donovan616add72023-10-11 08:45:11 +0000759 mutex_unlock(&ctrl->dhchap_auth_mutex);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200760 if (ret) {
761 chap->error = ret;
762 goto fail2;
763 }
764
765 /* DH-HMAC-CHAP Step 3: send reply */
766 dev_dbg(ctrl->device, "%s: qid %d send reply\n",
767 __func__, chap->qid);
768 ret = nvme_auth_set_dhchap_reply_data(ctrl, chap);
769 if (ret < 0) {
770 chap->error = ret;
771 goto fail2;
772 }
773
774 tl = ret;
775 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
776 if (ret) {
777 chap->error = ret;
778 goto fail2;
779 }
780
781 /* DH-HMAC-CHAP Step 4: receive success1 */
782 dev_dbg(ctrl->device, "%s: qid %d receive success1\n",
783 __func__, chap->qid);
784
Sagi Grimbergb7d604c2022-11-13 13:24:13 +0200785 memset(chap->buf, 0, CHAP_BUF_SIZE);
786 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE,
787 false);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200788 if (ret) {
789 dev_warn(ctrl->device,
790 "qid %d failed to receive success1, %s %d\n",
791 chap->qid, ret < 0 ? "error" : "nvme status", ret);
792 chap->error = ret;
793 return;
794 }
795 ret = nvme_auth_receive_validate(ctrl, chap->qid,
796 chap->buf, chap->transaction,
797 NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1);
798 if (ret) {
799 chap->status = ret;
Daniel Wagner0e34bd92024-04-30 15:19:28 +0200800 chap->error = -EKEYREJECTED;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200801 return;
802 }
803
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200804 mutex_lock(&ctrl->dhchap_auth_mutex);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200805 if (ctrl->ctrl_key) {
806 dev_dbg(ctrl->device,
807 "%s: qid %d controller response\n",
808 __func__, chap->qid);
809 ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap);
810 if (ret) {
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200811 mutex_unlock(&ctrl->dhchap_auth_mutex);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200812 chap->error = ret;
813 goto fail2;
814 }
815 }
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200816 mutex_unlock(&ctrl->dhchap_auth_mutex);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200817
818 ret = nvme_auth_process_dhchap_success1(ctrl, chap);
819 if (ret) {
820 /* Controller authentication failed */
Daniel Wagner0e34bd92024-04-30 15:19:28 +0200821 chap->error = -EKEYREJECTED;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200822 goto fail2;
823 }
824
Mark O'Donovanfc1e03e2023-10-25 10:51:24 +0000825 if (chap->bi_directional) {
Hannes Reineckef50fff72022-06-27 11:52:02 +0200826 /* DH-HMAC-CHAP Step 5: send success2 */
827 dev_dbg(ctrl->device, "%s: qid %d send success2\n",
828 __func__, chap->qid);
829 tl = nvme_auth_set_dhchap_success2_data(ctrl, chap);
830 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
831 if (ret)
832 chap->error = ret;
833 }
834 if (!ret) {
835 chap->error = 0;
836 return;
837 }
838
839fail2:
Mark O'Donovan38ce1572023-10-11 08:45:12 +0000840 if (chap->status == 0)
841 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200842 dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n",
843 __func__, chap->qid, chap->status);
844 tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap);
845 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true);
846 /*
847 * only update error if send failure2 failed and no other
848 * error had been set during authentication.
849 */
850 if (ret && !chap->error)
851 chap->error = ret;
852}
853
854int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid)
855{
856 struct nvme_dhchap_queue_context *chap;
857
858 if (!ctrl->host_key) {
859 dev_warn(ctrl->device, "qid %d: no key\n", qid);
860 return -ENOKEY;
861 }
862
863 if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) {
864 dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid);
865 return -ENOKEY;
866 }
867
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200868 chap = &ctrl->dhchap_ctxs[qid];
869 cancel_work_sync(&chap->auth_work);
Shin'ichiro Kawasakibd97a592023-01-31 18:26:44 +0900870 queue_work(nvme_auth_wq, &chap->auth_work);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200871 return 0;
872}
873EXPORT_SYMBOL_GPL(nvme_auth_negotiate);
874
875int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid)
876{
877 struct nvme_dhchap_queue_context *chap;
878 int ret;
879
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200880 chap = &ctrl->dhchap_ctxs[qid];
881 flush_work(&chap->auth_work);
882 ret = chap->error;
883 /* clear sensitive info */
884 nvme_auth_reset_dhchap(chap);
885 return ret;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200886}
887EXPORT_SYMBOL_GPL(nvme_auth_wait);
888
Sagi Grimberg0c999e62022-11-13 13:24:06 +0200889static void nvme_ctrl_auth_work(struct work_struct *work)
Hannes Reineckef50fff72022-06-27 11:52:02 +0200890{
891 struct nvme_ctrl *ctrl =
892 container_of(work, struct nvme_ctrl, dhchap_auth_work);
893 int ret, q;
894
Sagi Grimbergc7390f12022-11-13 13:24:08 +0200895 /*
896 * If the ctrl is no connected, bail as reconnect will handle
897 * authentication.
898 */
Keith Busch6d3c7fb2024-01-24 09:27:27 -0800899 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
Sagi Grimbergc7390f12022-11-13 13:24:08 +0200900 return;
901
Hannes Reineckef50fff72022-06-27 11:52:02 +0200902 /* Authenticate admin queue first */
903 ret = nvme_auth_negotiate(ctrl, 0);
904 if (ret) {
905 dev_warn(ctrl->device,
906 "qid 0: error %d setting up authentication\n", ret);
907 return;
908 }
909 ret = nvme_auth_wait(ctrl, 0);
910 if (ret) {
911 dev_warn(ctrl->device,
912 "qid 0: authentication failed\n");
913 return;
914 }
915
916 for (q = 1; q < ctrl->queue_count; q++) {
917 ret = nvme_auth_negotiate(ctrl, q);
918 if (ret) {
919 dev_warn(ctrl->device,
920 "qid %d: error %d setting up authentication\n",
921 q, ret);
922 break;
923 }
924 }
925
926 /*
927 * Failure is a soft-state; credentials remain valid until
928 * the controller terminates the connection.
929 */
Sagi Grimbergd061a1b2022-11-13 13:24:22 +0200930 for (q = 1; q < ctrl->queue_count; q++) {
931 ret = nvme_auth_wait(ctrl, q);
932 if (ret)
933 dev_warn(ctrl->device,
934 "qid %d: authentication failed\n", q);
935 }
Hannes Reineckef50fff72022-06-27 11:52:02 +0200936}
937
Sagi Grimberg193a8c72022-11-13 13:24:10 +0200938int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl)
Hannes Reineckef50fff72022-06-27 11:52:02 +0200939{
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200940 struct nvme_dhchap_queue_context *chap;
941 int i, ret;
Sagi Grimberg193a8c72022-11-13 13:24:10 +0200942
Hannes Reineckef50fff72022-06-27 11:52:02 +0200943 mutex_init(&ctrl->dhchap_auth_mutex);
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200944 INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200945 if (!ctrl->opts)
Sagi Grimberg193a8c72022-11-13 13:24:10 +0200946 return 0;
947 ret = nvme_auth_generate_key(ctrl->opts->dhchap_secret,
948 &ctrl->host_key);
949 if (ret)
950 return ret;
951 ret = nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret,
952 &ctrl->ctrl_key);
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200953 if (ret)
954 goto err_free_dhchap_secret;
955
956 if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret)
Sagi Grimberg76807fc2022-12-25 13:28:51 +0200957 return 0;
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200958
959 ctrl->dhchap_ctxs = kvcalloc(ctrl_max_dhchaps(ctrl),
960 sizeof(*chap), GFP_KERNEL);
961 if (!ctrl->dhchap_ctxs) {
962 ret = -ENOMEM;
963 goto err_free_dhchap_ctrl_secret;
Sagi Grimberg193a8c72022-11-13 13:24:10 +0200964 }
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200965
966 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) {
967 chap = &ctrl->dhchap_ctxs[i];
968 chap->qid = i;
969 chap->ctrl = ctrl;
970 INIT_WORK(&chap->auth_work, nvme_queue_auth_work);
971 }
972
973 return 0;
974err_free_dhchap_ctrl_secret:
975 nvme_auth_free_key(ctrl->ctrl_key);
976 ctrl->ctrl_key = NULL;
977err_free_dhchap_secret:
978 nvme_auth_free_key(ctrl->host_key);
979 ctrl->host_key = NULL;
Sagi Grimberg193a8c72022-11-13 13:24:10 +0200980 return ret;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200981}
982EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl);
983
984void nvme_auth_stop(struct nvme_ctrl *ctrl)
985{
Hannes Reineckef50fff72022-06-27 11:52:02 +0200986 cancel_work_sync(&ctrl->dhchap_auth_work);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200987}
988EXPORT_SYMBOL_GPL(nvme_auth_stop);
989
990void nvme_auth_free(struct nvme_ctrl *ctrl)
991{
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200992 int i;
Hannes Reineckef50fff72022-06-27 11:52:02 +0200993
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200994 if (ctrl->dhchap_ctxs) {
Sagi Grimberga2a00d22022-11-13 13:24:21 +0200995 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++)
996 nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]);
Sagi Grimbergaa36d712022-11-13 13:24:20 +0200997 kfree(ctrl->dhchap_ctxs);
Hannes Reineckef50fff72022-06-27 11:52:02 +0200998 }
Hannes Reineckef50fff72022-06-27 11:52:02 +0200999 if (ctrl->host_key) {
1000 nvme_auth_free_key(ctrl->host_key);
1001 ctrl->host_key = NULL;
1002 }
1003 if (ctrl->ctrl_key) {
1004 nvme_auth_free_key(ctrl->ctrl_key);
1005 ctrl->ctrl_key = NULL;
1006 }
1007}
1008EXPORT_SYMBOL_GPL(nvme_auth_free);
Sagi Grimberge481fc02022-11-15 17:08:06 +01001009
1010int __init nvme_init_auth(void)
1011{
Shin'ichiro Kawasakibd97a592023-01-31 18:26:44 +09001012 nvme_auth_wq = alloc_workqueue("nvme-auth-wq",
1013 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
1014 if (!nvme_auth_wq)
1015 return -ENOMEM;
1016
Sagi Grimberge481fc02022-11-15 17:08:06 +01001017 nvme_chap_buf_cache = kmem_cache_create("nvme-chap-buf-cache",
1018 CHAP_BUF_SIZE, 0, SLAB_HWCACHE_ALIGN, NULL);
1019 if (!nvme_chap_buf_cache)
Shin'ichiro Kawasakibd97a592023-01-31 18:26:44 +09001020 goto err_destroy_workqueue;
Sagi Grimberge481fc02022-11-15 17:08:06 +01001021
1022 nvme_chap_buf_pool = mempool_create(16, mempool_alloc_slab,
1023 mempool_free_slab, nvme_chap_buf_cache);
1024 if (!nvme_chap_buf_pool)
1025 goto err_destroy_chap_buf_cache;
1026
1027 return 0;
1028err_destroy_chap_buf_cache:
1029 kmem_cache_destroy(nvme_chap_buf_cache);
Shin'ichiro Kawasakibd97a592023-01-31 18:26:44 +09001030err_destroy_workqueue:
1031 destroy_workqueue(nvme_auth_wq);
Sagi Grimberge481fc02022-11-15 17:08:06 +01001032 return -ENOMEM;
1033}
1034
1035void __exit nvme_exit_auth(void)
1036{
1037 mempool_destroy(nvme_chap_buf_pool);
1038 kmem_cache_destroy(nvme_chap_buf_cache);
Shin'ichiro Kawasakibd97a592023-01-31 18:26:44 +09001039 destroy_workqueue(nvme_auth_wq);
Sagi Grimberge481fc02022-11-15 17:08:06 +01001040}