blob: 88cff309d8e4f06a18661f8c10f8ff318a835834 [file] [log] [blame]
Christoph Hellwigbc50ad72019-02-18 09:36:29 +01001// SPDX-License-Identifier: GPL-2.0
Christoph Hellwig21d34712015-11-26 09:08:36 +01002/*
3 * NVM Express device driver
4 * Copyright (c) 2011-2014, Intel Corporation.
Christoph Hellwig21d34712015-11-26 09:08:36 +01005 */
6
7#include <linux/blkdev.h>
8#include <linux/blk-mq.h>
Nick Bowlerc95b7082020-03-28 01:09:09 -04009#include <linux/compat.h>
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +010010#include <linux/delay.h>
Christoph Hellwig21d34712015-11-26 09:08:36 +010011#include <linux/errno.h>
Christoph Hellwig1673f1f2015-11-26 10:54:19 +010012#include <linux/hdreg.h>
Christoph Hellwig21d34712015-11-26 09:08:36 +010013#include <linux/kernel.h>
Christoph Hellwig5bae7f72015-11-28 15:39:07 +010014#include <linux/module.h>
Mikhail Skorzhinskii958f2a02019-07-04 09:59:18 +020015#include <linux/backing-dev.h>
Christoph Hellwig5bae7f72015-11-28 15:39:07 +010016#include <linux/list_sort.h>
Christoph Hellwig21d34712015-11-26 09:08:36 +010017#include <linux/slab.h>
18#include <linux/types.h>
Christoph Hellwig1673f1f2015-11-26 10:54:19 +010019#include <linux/pr.h>
20#include <linux/ptrace.h>
21#include <linux/nvme_ioctl.h>
Andy Lutomirskic5552fd2017-02-07 10:08:45 -080022#include <linux/pm_qos.h>
Christoph Hellwig1673f1f2015-11-26 10:54:19 +010023#include <asm/unaligned.h>
Christoph Hellwig21d34712015-11-26 09:08:36 +010024
25#include "nvme.h"
Sagi Grimberg038bd4c2016-06-13 16:45:28 +020026#include "fabrics.h"
Christoph Hellwig21d34712015-11-26 09:08:36 +010027
Hannes Reinecke35fe0d12019-07-24 15:47:55 +020028#define CREATE_TRACE_POINTS
29#include "trace.h"
30
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +010031#define NVME_MINORS (1U << MINORBITS)
32
Marc Olson8ae4e442017-09-06 17:23:56 -070033unsigned int admin_timeout = 60;
34module_param(admin_timeout, uint, 0644);
Ming Linba0ba7d2016-02-10 10:03:30 -080035MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
Ming Lin576d55d2016-02-10 10:03:32 -080036EXPORT_SYMBOL_GPL(admin_timeout);
Ming Linba0ba7d2016-02-10 10:03:30 -080037
Marc Olson8ae4e442017-09-06 17:23:56 -070038unsigned int nvme_io_timeout = 30;
39module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
Ming Linba0ba7d2016-02-10 10:03:30 -080040MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
Ming Lin576d55d2016-02-10 10:03:32 -080041EXPORT_SYMBOL_GPL(nvme_io_timeout);
Ming Linba0ba7d2016-02-10 10:03:30 -080042
Christoph Hellwigb3b1b0b2017-06-12 18:30:51 +020043static unsigned char shutdown_timeout = 5;
Ming Linba0ba7d2016-02-10 10:03:30 -080044module_param(shutdown_timeout, byte, 0644);
45MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
46
Christoph Hellwig44e44b22017-04-05 19:18:11 +020047static u8 nvme_max_retries = 5;
48module_param_named(max_retries, nvme_max_retries, byte, 0644);
Keith Buschf80ec962016-07-12 16:20:31 -070049MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
Christoph Hellwig5bae7f72015-11-28 15:39:07 +010050
Kai-Heng Feng9947d6a2017-06-07 15:25:43 +080051static unsigned long default_ps_max_latency_us = 100000;
Andy Lutomirskic5552fd2017-02-07 10:08:45 -080052module_param(default_ps_max_latency_us, ulong, 0644);
53MODULE_PARM_DESC(default_ps_max_latency_us,
54 "max power saving latency for new devices; use PM QOS to change per device");
55
Andy Lutomirskic35e30b2017-04-21 16:19:24 -070056static bool force_apst;
57module_param(force_apst, bool, 0644);
58MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
59
Jens Axboef5d11842017-06-27 12:03:06 -060060static bool streams;
61module_param(streams, bool, 0644);
62MODULE_PARM_DESC(streams, "turn on support for Streams write directives");
63
Roy Shtermanb227c592018-01-14 12:39:02 +020064/*
65 * nvme_wq - hosts nvme related works that are not reset or delete
66 * nvme_reset_wq - hosts nvme reset works
67 * nvme_delete_wq - hosts nvme delete works
68 *
Nigel Kirkland97b25122020-02-10 16:01:45 -080069 * nvme_wq will host works such as scan, aen handling, fw activation,
70 * keep-alive, periodic reconnects etc. nvme_reset_wq
Roy Shtermanb227c592018-01-14 12:39:02 +020071 * runs reset works which also flush works hosted on nvme_wq for
72 * serialization purposes. nvme_delete_wq host controller deletion
73 * works which flush reset works for serialization.
74 */
Sagi Grimberg9a6327d2017-06-07 20:31:55 +020075struct workqueue_struct *nvme_wq;
76EXPORT_SYMBOL_GPL(nvme_wq);
77
Roy Shtermanb227c592018-01-14 12:39:02 +020078struct workqueue_struct *nvme_reset_wq;
79EXPORT_SYMBOL_GPL(nvme_reset_wq);
80
81struct workqueue_struct *nvme_delete_wq;
82EXPORT_SYMBOL_GPL(nvme_delete_wq);
83
Christoph Hellwigab9e00c2017-11-09 13:48:55 +010084static LIST_HEAD(nvme_subsystems);
85static DEFINE_MUTEX(nvme_subsystems_lock);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +010086
Christoph Hellwig9843f682017-10-18 13:10:01 +020087static DEFINE_IDA(nvme_instance_ida);
Christoph Hellwiga6a51492017-10-18 16:59:25 +020088static dev_t nvme_chr_devt;
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +010089static struct class *nvme_class;
Christoph Hellwigab9e00c2017-11-09 13:48:55 +010090static struct class *nvme_subsys_class;
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +010091
Keith Busch240e6ee2020-06-29 12:06:41 -070092static int _nvme_revalidate_disk(struct gendisk *disk);
Jianchao Wang12d9f072018-05-04 16:01:57 +080093static void nvme_put_subsystem(struct nvme_subsystem *subsys);
Scott Bauercf39a6b2018-06-29 13:03:28 -060094static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
95 unsigned nsid);
96
97static void nvme_set_queue_dying(struct nvme_ns *ns)
98{
99 /*
100 * Revalidating a dead namespace sets capacity to 0. This will end
101 * buffered writers dirtying pages that can't be synced.
102 */
Christoph Hellwig3913f4f2020-07-08 16:18:27 +0200103 if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))
Scott Bauercf39a6b2018-06-29 13:03:28 -0600104 return;
Scott Bauercf39a6b2018-06-29 13:03:28 -0600105 blk_set_queue_dying(ns->queue);
106 /* Forcibly unquiesce queues to avoid blocking dispatch */
107 blk_mq_unquiesce_queue(ns->queue);
Balbir Singhb2247262019-09-18 00:27:20 +0000108 /*
109 * Revalidate after unblocking dispatchers that may be holding bd_butex
110 */
111 revalidate_disk(ns->disk);
Scott Bauercf39a6b2018-06-29 13:03:28 -0600112}
Christoph Hellwig1673f1f2015-11-26 10:54:19 +0100113
Christoph Hellwig50e8d8e2018-05-25 18:15:47 +0200114static void nvme_queue_scan(struct nvme_ctrl *ctrl)
115{
116 /*
117 * Only new queue scan work when admin and IO queues are both alive
118 */
Keith Busch5d02a5c2019-09-03 09:22:24 -0600119 if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
Christoph Hellwig50e8d8e2018-05-25 18:15:47 +0200120 queue_work(nvme_wq, &ctrl->scan_work);
121}
122
Keith Busch4c75f872019-09-06 11:23:08 -0600123/*
124 * Use this function to proceed with scheduling reset_work for a controller
125 * that had previously been set to the resetting state. This is intended for
126 * code paths that can't be interrupted by other reset attempts. A hot removal
127 * may prevent this from succeeding.
128 */
Keith Buschc1ac9a4b2019-09-04 10:06:11 -0600129int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
Keith Busch4c75f872019-09-06 11:23:08 -0600130{
131 if (ctrl->state != NVME_CTRL_RESETTING)
132 return -EBUSY;
133 if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
134 return -EBUSY;
135 return 0;
136}
Keith Buschc1ac9a4b2019-09-04 10:06:11 -0600137EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
Keith Busch4c75f872019-09-06 11:23:08 -0600138
Christoph Hellwigd86c4d82017-06-15 15:41:08 +0200139int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
140{
141 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
142 return -EBUSY;
Roy Shtermanb227c592018-01-14 12:39:02 +0200143 if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
Christoph Hellwigd86c4d82017-06-15 15:41:08 +0200144 return -EBUSY;
145 return 0;
146}
147EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
148
Sagi Grimberg79c48cc2018-01-14 12:39:00 +0200149int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
Christoph Hellwigd86c4d82017-06-15 15:41:08 +0200150{
151 int ret;
152
153 ret = nvme_reset_ctrl(ctrl);
Nitzan Carmi8000d1f2018-01-17 11:01:14 +0000154 if (!ret) {
Christoph Hellwigd86c4d82017-06-15 15:41:08 +0200155 flush_work(&ctrl->reset_work);
Keith Busch5d02a5c2019-09-03 09:22:24 -0600156 if (ctrl->state != NVME_CTRL_LIVE)
Nitzan Carmi8000d1f2018-01-17 11:01:14 +0000157 ret = -ENETRESET;
158 }
159
Christoph Hellwigd86c4d82017-06-15 15:41:08 +0200160 return ret;
161}
Sagi Grimberg79c48cc2018-01-14 12:39:00 +0200162EXPORT_SYMBOL_GPL(nvme_reset_ctrl_sync);
Christoph Hellwigd86c4d82017-06-15 15:41:08 +0200163
Bart Van Asschea686ed72019-02-14 14:50:56 -0800164static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
Christoph Hellwigc5017e82017-10-29 10:44:29 +0200165{
Max Gurtovoy77d06122018-03-11 17:46:06 +0200166 dev_info(ctrl->device,
167 "Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn);
168
Sagi Grimberg40546372017-10-29 14:21:02 +0200169 flush_work(&ctrl->reset_work);
Christoph Hellwig6cd53d12017-10-29 10:44:31 +0200170 nvme_stop_ctrl(ctrl);
171 nvme_remove_namespaces(ctrl);
Christoph Hellwigc5017e82017-10-29 10:44:29 +0200172 ctrl->ops->delete_ctrl(ctrl);
Christoph Hellwig6cd53d12017-10-29 10:44:31 +0200173 nvme_uninit_ctrl(ctrl);
Christoph Hellwigc5017e82017-10-29 10:44:29 +0200174}
175
Bart Van Asschea686ed72019-02-14 14:50:56 -0800176static void nvme_delete_ctrl_work(struct work_struct *work)
177{
178 struct nvme_ctrl *ctrl =
179 container_of(work, struct nvme_ctrl, delete_work);
180
181 nvme_do_delete_ctrl(ctrl);
182}
183
Christoph Hellwigc5017e82017-10-29 10:44:29 +0200184int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
185{
186 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
187 return -EBUSY;
Roy Shtermanb227c592018-01-14 12:39:02 +0200188 if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
Christoph Hellwigc5017e82017-10-29 10:44:29 +0200189 return -EBUSY;
190 return 0;
191}
192EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
193
Israel Rukshin6721c182020-03-24 17:29:39 +0200194static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
Christoph Hellwigc5017e82017-10-29 10:44:29 +0200195{
Christoph Hellwigc5017e82017-10-29 10:44:29 +0200196 /*
Yufen Yu01fc08f2019-03-13 18:54:58 +0100197 * Keep a reference until nvme_do_delete_ctrl() complete,
198 * since ->delete_ctrl can free the controller.
Christoph Hellwigc5017e82017-10-29 10:44:29 +0200199 */
200 nvme_get_ctrl(ctrl);
Israel Rukshin6721c182020-03-24 17:29:39 +0200201 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
Bart Van Asscheb9c77582019-02-14 14:50:57 -0800202 nvme_do_delete_ctrl(ctrl);
Christoph Hellwigc5017e82017-10-29 10:44:29 +0200203 nvme_put_ctrl(ctrl);
Christoph Hellwigc5017e82017-10-29 10:44:29 +0200204}
Christoph Hellwigc5017e82017-10-29 10:44:29 +0200205
Sagi Grimberg2f9c1732019-08-29 12:53:15 -0700206static blk_status_t nvme_error_status(u16 status)
Christoph Hellwig27fa9bc2017-04-20 16:02:57 +0200207{
Sagi Grimberg2f9c1732019-08-29 12:53:15 -0700208 switch (status & 0x7ff) {
Christoph Hellwig27fa9bc2017-04-20 16:02:57 +0200209 case NVME_SC_SUCCESS:
Christoph Hellwig2a842ac2017-06-03 09:38:04 +0200210 return BLK_STS_OK;
Christoph Hellwig27fa9bc2017-04-20 16:02:57 +0200211 case NVME_SC_CAP_EXCEEDED:
Christoph Hellwig2a842ac2017-06-03 09:38:04 +0200212 return BLK_STS_NOSPC;
Keith Busche96fef22018-01-09 12:04:14 -0700213 case NVME_SC_LBA_RANGE:
Keith Busch35038bf2019-12-06 04:50:44 +0900214 case NVME_SC_CMD_INTERRUPTED:
215 case NVME_SC_NS_NOT_READY:
Keith Busche96fef22018-01-09 12:04:14 -0700216 return BLK_STS_TARGET;
217 case NVME_SC_BAD_ATTRIBUTES:
Junxiong Guane02ab022017-04-21 12:59:07 +0200218 case NVME_SC_ONCS_NOT_SUPPORTED:
Keith Busche96fef22018-01-09 12:04:14 -0700219 case NVME_SC_INVALID_OPCODE:
220 case NVME_SC_INVALID_FIELD:
221 case NVME_SC_INVALID_NS:
Christoph Hellwig2a842ac2017-06-03 09:38:04 +0200222 return BLK_STS_NOTSUPP;
Junxiong Guane02ab022017-04-21 12:59:07 +0200223 case NVME_SC_WRITE_FAULT:
224 case NVME_SC_READ_ERROR:
225 case NVME_SC_UNWRITTEN_BLOCK:
Christoph Hellwiga751da32017-08-22 10:17:03 +0200226 case NVME_SC_ACCESS_DENIED:
227 case NVME_SC_READ_ONLY:
Keith Busche96fef22018-01-09 12:04:14 -0700228 case NVME_SC_COMPARE_FAILED:
Christoph Hellwig2a842ac2017-06-03 09:38:04 +0200229 return BLK_STS_MEDIUM;
Christoph Hellwiga751da32017-08-22 10:17:03 +0200230 case NVME_SC_GUARD_CHECK:
231 case NVME_SC_APPTAG_CHECK:
232 case NVME_SC_REFTAG_CHECK:
233 case NVME_SC_INVALID_PI:
234 return BLK_STS_PROTECTION;
235 case NVME_SC_RESERVATION_CONFLICT:
236 return BLK_STS_NEXUS;
Sagi Grimberg1c0d12c2019-08-02 18:04:12 -0700237 case NVME_SC_HOST_PATH_ERROR:
238 return BLK_STS_TRANSPORT;
Christoph Hellwig2a842ac2017-06-03 09:38:04 +0200239 default:
240 return BLK_STS_IOERR;
Christoph Hellwig27fa9bc2017-04-20 16:02:57 +0200241 }
242}
Christoph Hellwig27fa9bc2017-04-20 16:02:57 +0200243
Christoph Hellwigf6324b12017-04-05 19:18:09 +0200244static inline bool nvme_req_needs_retry(struct request *req)
Christoph Hellwig77f02a72017-03-30 13:41:32 +0200245{
Christoph Hellwigf6324b12017-04-05 19:18:09 +0200246 if (blk_noretry_request(req))
247 return false;
Christoph Hellwig27fa9bc2017-04-20 16:02:57 +0200248 if (nvme_req(req)->status & NVME_SC_DNR)
Christoph Hellwigf6324b12017-04-05 19:18:09 +0200249 return false;
Christoph Hellwig44e44b22017-04-05 19:18:11 +0200250 if (nvme_req(req)->retries >= nvme_max_retries)
Christoph Hellwigf6324b12017-04-05 19:18:09 +0200251 return false;
252 return true;
Christoph Hellwig77f02a72017-03-30 13:41:32 +0200253}
254
Keith Busch49cd84b2018-11-27 09:40:57 -0700255static void nvme_retry_req(struct request *req)
256{
257 struct nvme_ns *ns = req->q->queuedata;
258 unsigned long delay = 0;
259 u16 crd;
260
261 /* The mask and shift result must be <= 3 */
262 crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
263 if (ns && crd)
264 delay = ns->ctrl->crdt[crd - 1] * 100;
265
266 nvme_req(req)->retries++;
267 blk_mq_requeue_request(req, false);
268 blk_mq_delay_kick_requeue_list(req->q, delay);
269}
270
Christoph Hellwig77f02a72017-03-30 13:41:32 +0200271void nvme_complete_rq(struct request *req)
272{
Sagi Grimberg2f9c1732019-08-29 12:53:15 -0700273 blk_status_t status = nvme_error_status(nvme_req(req)->status);
Keith Busch908e4562018-01-09 12:04:15 -0700274
Johannes Thumshirnca5554a2018-01-26 11:21:38 +0100275 trace_nvme_complete_rq(req);
276
Max Gurtovoy16686f32019-10-13 19:57:36 +0300277 nvme_cleanup_cmd(req);
278
Sagi Grimberg6e3ca03e2018-11-02 10:28:15 -0700279 if (nvme_req(req)->ctrl->kas)
280 nvme_req(req)->ctrl->comp_seen = true;
281
Keith Busch908e4562018-01-09 12:04:15 -0700282 if (unlikely(status != BLK_STS_OK && nvme_req_needs_retry(req))) {
John Meneghini764e9332020-02-20 10:05:38 +0900283 if ((req->cmd_flags & REQ_NVME_MPATH) && nvme_failover_req(req))
Christoph Hellwig32acab32017-11-02 12:59:30 +0100284 return;
Christoph Hellwig32acab32017-11-02 12:59:30 +0100285
286 if (!blk_queue_dying(req->q)) {
Keith Busch49cd84b2018-11-27 09:40:57 -0700287 nvme_retry_req(req);
Christoph Hellwig32acab32017-11-02 12:59:30 +0100288 return;
289 }
Keith Busch240e6ee2020-06-29 12:06:41 -0700290 } else if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
291 req_op(req) == REQ_OP_ZONE_APPEND) {
292 req->__sector = nvme_lba_to_sect(req->q->queuedata,
293 le64_to_cpu(nvme_req(req)->result.u64));
Christoph Hellwig77f02a72017-03-30 13:41:32 +0200294 }
Hannes Reinecke35fe0d12019-07-24 15:47:55 +0200295
296 nvme_trace_bio_complete(req, status);
Keith Busch908e4562018-01-09 12:04:15 -0700297 blk_mq_end_request(req, status);
Christoph Hellwig77f02a72017-03-30 13:41:32 +0200298}
299EXPORT_SYMBOL_GPL(nvme_complete_rq);
300
Jens Axboe7baa8572018-11-08 10:24:07 -0700301bool nvme_cancel_request(struct request *req, void *data, bool reserved)
Ming Linc55a2fd2016-05-18 14:05:02 -0700302{
Ming Linc55a2fd2016-05-18 14:05:02 -0700303 dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
304 "Cancelling I/O %d", req->tag);
305
Ming Lei78ca4072019-07-24 11:48:41 +0800306 /* don't abort one completed request */
307 if (blk_mq_request_completed(req))
308 return true;
309
Max Gurtovoy2dc39472019-10-13 19:57:35 +0300310 nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
Christoph Hellwig15f73f52020-06-11 08:44:47 +0200311 blk_mq_complete_request(req);
Jens Axboe7baa8572018-11-08 10:24:07 -0700312 return true;
Ming Linc55a2fd2016-05-18 14:05:02 -0700313}
314EXPORT_SYMBOL_GPL(nvme_cancel_request);
315
Christoph Hellwigbb8d2612016-04-26 13:51:57 +0200316bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
317 enum nvme_ctrl_state new_state)
318{
Gabriel Krisman Bertazif6b6a282016-07-29 16:15:18 -0300319 enum nvme_ctrl_state old_state;
Christoph Hellwig0a72bbb2017-08-22 11:42:24 +0200320 unsigned long flags;
Christoph Hellwigbb8d2612016-04-26 13:51:57 +0200321 bool changed = false;
322
Christoph Hellwig0a72bbb2017-08-22 11:42:24 +0200323 spin_lock_irqsave(&ctrl->lock, flags);
Gabriel Krisman Bertazif6b6a282016-07-29 16:15:18 -0300324
325 old_state = ctrl->state;
Christoph Hellwigbb8d2612016-04-26 13:51:57 +0200326 switch (new_state) {
327 case NVME_CTRL_LIVE:
328 switch (old_state) {
Christoph Hellwig7d2e8002016-06-13 16:45:22 +0200329 case NVME_CTRL_NEW:
Christoph Hellwigbb8d2612016-04-26 13:51:57 +0200330 case NVME_CTRL_RESETTING:
Max Gurtovoyad6a0a52018-01-31 18:31:24 +0200331 case NVME_CTRL_CONNECTING:
Christoph Hellwigbb8d2612016-04-26 13:51:57 +0200332 changed = true;
333 /* FALLTHRU */
334 default:
335 break;
336 }
337 break;
338 case NVME_CTRL_RESETTING:
339 switch (old_state) {
340 case NVME_CTRL_NEW:
341 case NVME_CTRL_LIVE:
Christoph Hellwigdef61ec2016-07-06 21:55:49 +0900342 changed = true;
343 /* FALLTHRU */
344 default:
345 break;
346 }
347 break;
Max Gurtovoyad6a0a52018-01-31 18:31:24 +0200348 case NVME_CTRL_CONNECTING:
Christoph Hellwigdef61ec2016-07-06 21:55:49 +0900349 switch (old_state) {
Max Gurtovoyb754a322018-01-31 18:31:25 +0200350 case NVME_CTRL_NEW:
James Smart3cec7f92017-10-25 16:43:13 -0700351 case NVME_CTRL_RESETTING:
Christoph Hellwigbb8d2612016-04-26 13:51:57 +0200352 changed = true;
353 /* FALLTHRU */
354 default:
355 break;
356 }
357 break;
358 case NVME_CTRL_DELETING:
359 switch (old_state) {
360 case NVME_CTRL_LIVE:
361 case NVME_CTRL_RESETTING:
Max Gurtovoyad6a0a52018-01-31 18:31:24 +0200362 case NVME_CTRL_CONNECTING:
Christoph Hellwigbb8d2612016-04-26 13:51:57 +0200363 changed = true;
364 /* FALLTHRU */
365 default:
366 break;
367 }
368 break;
Sagi Grimbergecca390e2020-07-22 16:32:19 -0700369 case NVME_CTRL_DELETING_NOIO:
370 switch (old_state) {
371 case NVME_CTRL_DELETING:
372 case NVME_CTRL_DEAD:
373 changed = true;
374 /* FALLTHRU */
375 default:
376 break;
377 }
378 break;
Keith Busch0ff9d4e2016-05-12 08:37:14 -0600379 case NVME_CTRL_DEAD:
380 switch (old_state) {
381 case NVME_CTRL_DELETING:
382 changed = true;
383 /* FALLTHRU */
384 default:
385 break;
386 }
387 break;
Christoph Hellwigbb8d2612016-04-26 13:51:57 +0200388 default:
389 break;
390 }
Christoph Hellwigbb8d2612016-04-26 13:51:57 +0200391
Keith Buschc1ac9a4b2019-09-04 10:06:11 -0600392 if (changed) {
Christoph Hellwigbb8d2612016-04-26 13:51:57 +0200393 ctrl->state = new_state;
Keith Buschc1ac9a4b2019-09-04 10:06:11 -0600394 wake_up_all(&ctrl->state_wq);
395 }
Christoph Hellwigbb8d2612016-04-26 13:51:57 +0200396
Christoph Hellwig0a72bbb2017-08-22 11:42:24 +0200397 spin_unlock_irqrestore(&ctrl->lock, flags);
Christoph Hellwig32acab32017-11-02 12:59:30 +0100398 if (changed && ctrl->state == NVME_CTRL_LIVE)
399 nvme_kick_requeue_lists(ctrl);
Christoph Hellwigbb8d2612016-04-26 13:51:57 +0200400 return changed;
401}
402EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
403
Keith Buschc1ac9a4b2019-09-04 10:06:11 -0600404/*
405 * Returns true for sink states that can't ever transition back to live.
406 */
407static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
408{
409 switch (ctrl->state) {
410 case NVME_CTRL_NEW:
411 case NVME_CTRL_LIVE:
412 case NVME_CTRL_RESETTING:
413 case NVME_CTRL_CONNECTING:
414 return false;
415 case NVME_CTRL_DELETING:
Sagi Grimbergecca390e2020-07-22 16:32:19 -0700416 case NVME_CTRL_DELETING_NOIO:
Keith Buschc1ac9a4b2019-09-04 10:06:11 -0600417 case NVME_CTRL_DEAD:
418 return true;
419 default:
420 WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
421 return true;
422 }
423}
424
425/*
426 * Waits for the controller state to be resetting, or returns false if it is
427 * not possible to ever transition to that state.
428 */
429bool nvme_wait_reset(struct nvme_ctrl *ctrl)
430{
431 wait_event(ctrl->state_wq,
432 nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
433 nvme_state_terminal(ctrl));
434 return ctrl->state == NVME_CTRL_RESETTING;
435}
436EXPORT_SYMBOL_GPL(nvme_wait_reset);
437
Christoph Hellwiged754e52017-11-09 13:50:43 +0100438static void nvme_free_ns_head(struct kref *ref)
439{
440 struct nvme_ns_head *head =
441 container_of(ref, struct nvme_ns_head, ref);
442
Christoph Hellwig32acab32017-11-02 12:59:30 +0100443 nvme_mpath_remove_disk(head);
Christoph Hellwiged754e52017-11-09 13:50:43 +0100444 ida_simple_remove(&head->subsys->ns_ida, head->instance);
Paul E. McKenneyf5ad3992019-02-13 13:54:37 -0800445 cleanup_srcu_struct(&head->srcu);
Jianchao Wang12d9f072018-05-04 16:01:57 +0800446 nvme_put_subsystem(head->subsys);
Christoph Hellwiged754e52017-11-09 13:50:43 +0100447 kfree(head);
448}
449
450static void nvme_put_ns_head(struct nvme_ns_head *head)
451{
452 kref_put(&head->ref, nvme_free_ns_head);
453}
454
Christoph Hellwig1673f1f2015-11-26 10:54:19 +0100455static void nvme_free_ns(struct kref *kref)
456{
457 struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
458
Matias Bjørlingb0b4e092016-09-16 14:25:07 +0200459 if (ns->ndev)
460 nvme_nvm_unregister(ns);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +0100461
Christoph Hellwig1673f1f2015-11-26 10:54:19 +0100462 put_disk(ns->disk);
Christoph Hellwiged754e52017-11-09 13:50:43 +0100463 nvme_put_ns_head(ns->head);
Keith Busch075790e2016-02-24 09:15:53 -0700464 nvme_put_ctrl(ns->ctrl);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +0100465 kfree(ns);
466}
467
Logan Gunthorpe24493b82020-07-24 11:25:16 -0600468void nvme_put_ns(struct nvme_ns *ns)
Christoph Hellwig1673f1f2015-11-26 10:54:19 +0100469{
470 kref_put(&ns->kref, nvme_free_ns);
471}
Logan Gunthorpe24493b82020-07-24 11:25:16 -0600472EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +0100473
James Smartbb06ec312018-04-12 09:16:15 -0600474static inline void nvme_clear_nvme_request(struct request *req)
475{
476 if (!(req->rq_flags & RQF_DONTPREP)) {
477 nvme_req(req)->retries = 0;
478 nvme_req(req)->flags = 0;
479 req->rq_flags |= RQF_DONTPREP;
480 }
481}
482
Christoph Hellwig41609822015-11-20 09:00:02 +0100483struct request *nvme_alloc_request(struct request_queue *q,
Bart Van Assche9a95e4e2017-11-09 10:49:59 -0800484 struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid)
Christoph Hellwig21d34712015-11-26 09:08:36 +0100485{
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100486 unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
Christoph Hellwig21d34712015-11-26 09:08:36 +0100487 struct request *req;
Christoph Hellwig21d34712015-11-26 09:08:36 +0100488
Christoph Hellwigeb71f432016-06-13 16:45:23 +0200489 if (qid == NVME_QID_ANY) {
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100490 req = blk_mq_alloc_request(q, op, flags);
Christoph Hellwigeb71f432016-06-13 16:45:23 +0200491 } else {
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100492 req = blk_mq_alloc_request_hctx(q, op, flags,
Christoph Hellwigeb71f432016-06-13 16:45:23 +0200493 qid ? qid - 1 : 0);
494 }
Christoph Hellwig21d34712015-11-26 09:08:36 +0100495 if (IS_ERR(req))
Christoph Hellwig41609822015-11-20 09:00:02 +0100496 return req;
Christoph Hellwig21d34712015-11-26 09:08:36 +0100497
Christoph Hellwig21d34712015-11-26 09:08:36 +0100498 req->cmd_flags |= REQ_FAILFAST_DRIVER;
James Smartbb06ec312018-04-12 09:16:15 -0600499 nvme_clear_nvme_request(req);
Christoph Hellwigd49187e2016-11-10 07:32:33 -0800500 nvme_req(req)->cmd = cmd;
Christoph Hellwig21d34712015-11-26 09:08:36 +0100501
Christoph Hellwig41609822015-11-20 09:00:02 +0100502 return req;
503}
Ming Lin576d55d2016-02-10 10:03:32 -0800504EXPORT_SYMBOL_GPL(nvme_alloc_request);
Christoph Hellwig41609822015-11-20 09:00:02 +0100505
Jens Axboef5d11842017-06-27 12:03:06 -0600506static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
507{
508 struct nvme_command c;
509
510 memset(&c, 0, sizeof(c));
511
512 c.directive.opcode = nvme_admin_directive_send;
Arnav Dawn62346ea2017-07-12 16:11:53 +0530513 c.directive.nsid = cpu_to_le32(NVME_NSID_ALL);
Jens Axboef5d11842017-06-27 12:03:06 -0600514 c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE;
515 c.directive.dtype = NVME_DIR_IDENTIFY;
516 c.directive.tdtype = NVME_DIR_STREAMS;
517 c.directive.endir = enable ? NVME_DIR_ENDIR : 0;
518
519 return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0);
520}
521
522static int nvme_disable_streams(struct nvme_ctrl *ctrl)
523{
524 return nvme_toggle_streams(ctrl, false);
525}
526
527static int nvme_enable_streams(struct nvme_ctrl *ctrl)
528{
529 return nvme_toggle_streams(ctrl, true);
530}
531
532static int nvme_get_stream_params(struct nvme_ctrl *ctrl,
533 struct streams_directive_params *s, u32 nsid)
534{
535 struct nvme_command c;
536
537 memset(&c, 0, sizeof(c));
538 memset(s, 0, sizeof(*s));
539
540 c.directive.opcode = nvme_admin_directive_recv;
541 c.directive.nsid = cpu_to_le32(nsid);
Keith Busch71fb90e2020-04-03 09:24:01 -0700542 c.directive.numd = cpu_to_le32(nvme_bytes_to_numd(sizeof(*s)));
Jens Axboef5d11842017-06-27 12:03:06 -0600543 c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM;
544 c.directive.dtype = NVME_DIR_STREAMS;
545
546 return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s));
547}
548
549static int nvme_configure_directives(struct nvme_ctrl *ctrl)
550{
551 struct streams_directive_params s;
552 int ret;
553
554 if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES))
555 return 0;
556 if (!streams)
557 return 0;
558
559 ret = nvme_enable_streams(ctrl);
560 if (ret)
561 return ret;
562
Arnav Dawn62346ea2017-07-12 16:11:53 +0530563 ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
Jens Axboef5d11842017-06-27 12:03:06 -0600564 if (ret)
Wu Bo84e4c202020-05-13 16:18:13 +0800565 goto out_disable_stream;
Jens Axboef5d11842017-06-27 12:03:06 -0600566
567 ctrl->nssa = le16_to_cpu(s.nssa);
568 if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) {
569 dev_info(ctrl->device, "too few streams (%u) available\n",
570 ctrl->nssa);
Wu Bo84e4c202020-05-13 16:18:13 +0800571 goto out_disable_stream;
Jens Axboef5d11842017-06-27 12:03:06 -0600572 }
573
Chaitanya Kulkarnia87835e2020-06-01 19:41:11 -0700574 ctrl->nr_streams = min_t(u16, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
Jens Axboef5d11842017-06-27 12:03:06 -0600575 dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams);
576 return 0;
Wu Bo84e4c202020-05-13 16:18:13 +0800577
578out_disable_stream:
579 nvme_disable_streams(ctrl);
580 return ret;
Jens Axboef5d11842017-06-27 12:03:06 -0600581}
582
583/*
584 * Check if 'req' has a write hint associated with it. If it does, assign
585 * a valid namespace stream to the write.
586 */
587static void nvme_assign_write_stream(struct nvme_ctrl *ctrl,
588 struct request *req, u16 *control,
589 u32 *dsmgmt)
590{
591 enum rw_hint streamid = req->write_hint;
592
593 if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE)
594 streamid = 0;
595 else {
596 streamid--;
597 if (WARN_ON_ONCE(streamid > ctrl->nr_streams))
598 return;
599
600 *control |= NVME_RW_DTYPE_STREAMS;
601 *dsmgmt |= streamid << 16;
602 }
603
604 if (streamid < ARRAY_SIZE(req->q->write_hints))
605 req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9;
606}
607
Logan Gunthorpe2bf5d3b2020-07-24 11:25:12 -0600608static void nvme_setup_passthrough(struct request *req,
609 struct nvme_command *cmd)
610{
611 memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
612 /* passthru commands should let the driver set the SGL flags */
613 cmd->common.flags &= ~NVME_CMD_SGL_ALL;
614}
615
Ming Lin8093f7c2016-04-12 13:10:14 -0600616static inline void nvme_setup_flush(struct nvme_ns *ns,
617 struct nvme_command *cmnd)
618{
Ming Lin8093f7c2016-04-12 13:10:14 -0600619 cmnd->common.opcode = nvme_cmd_flush;
Christoph Hellwiged754e52017-11-09 13:50:43 +0100620 cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
Ming Lin8093f7c2016-04-12 13:10:14 -0600621}
622
Christoph Hellwigfc17b652017-06-03 09:38:05 +0200623static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
Ming Lin8093f7c2016-04-12 13:10:14 -0600624 struct nvme_command *cmnd)
625{
Christoph Hellwigb35ba012017-02-08 14:46:50 +0100626 unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
Ming Lin8093f7c2016-04-12 13:10:14 -0600627 struct nvme_dsm_range *range;
Christoph Hellwigb35ba012017-02-08 14:46:50 +0100628 struct bio *bio;
Ming Lin8093f7c2016-04-12 13:10:14 -0600629
Eduard Hasenleithner530436c2019-11-12 21:55:01 +0100630 /*
631 * Some devices do not consider the DSM 'Number of Ranges' field when
632 * determining how much data to DMA. Always allocate memory for maximum
633 * number of segments to prevent device reading beyond end of buffer.
634 */
635 static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
636
637 range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
Jens Axboecb5b7262018-12-12 09:18:11 -0700638 if (!range) {
639 /*
640 * If we fail allocation our range, fallback to the controller
641 * discard page. If that's also busy, it's safe to return
642 * busy, as we know we can make progress once that's freed.
643 */
644 if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
645 return BLK_STS_RESOURCE;
646
647 range = page_address(ns->ctrl->discard_page);
648 }
Ming Lin8093f7c2016-04-12 13:10:14 -0600649
Christoph Hellwigb35ba012017-02-08 14:46:50 +0100650 __rq_for_each_bio(bio, req) {
Damien Le Moal314d48d2019-10-21 12:40:03 +0900651 u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
Christoph Hellwigb35ba012017-02-08 14:46:50 +0100652 u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
653
Keith Busch8cb6af72018-01-31 17:01:58 -0700654 if (n < segments) {
655 range[n].cattr = cpu_to_le32(0);
656 range[n].nlb = cpu_to_le32(nlb);
657 range[n].slba = cpu_to_le64(slba);
658 }
Christoph Hellwigb35ba012017-02-08 14:46:50 +0100659 n++;
660 }
661
662 if (WARN_ON_ONCE(n != segments)) {
Jens Axboecb5b7262018-12-12 09:18:11 -0700663 if (virt_to_page(range) == ns->ctrl->discard_page)
664 clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
665 else
666 kfree(range);
Christoph Hellwigfc17b652017-06-03 09:38:05 +0200667 return BLK_STS_IOERR;
Christoph Hellwigb35ba012017-02-08 14:46:50 +0100668 }
Ming Lin8093f7c2016-04-12 13:10:14 -0600669
Ming Lin8093f7c2016-04-12 13:10:14 -0600670 cmnd->dsm.opcode = nvme_cmd_dsm;
Christoph Hellwiged754e52017-11-09 13:50:43 +0100671 cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
Christoph Hellwigf1dd03a2017-03-31 17:00:05 +0200672 cmnd->dsm.nr = cpu_to_le32(segments - 1);
Ming Lin8093f7c2016-04-12 13:10:14 -0600673 cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
674
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700675 req->special_vec.bv_page = virt_to_page(range);
676 req->special_vec.bv_offset = offset_in_page(range);
Eduard Hasenleithner530436c2019-11-12 21:55:01 +0100677 req->special_vec.bv_len = alloc_size;
Christoph Hellwigf9d03f92016-12-08 15:20:32 -0700678 req->rq_flags |= RQF_SPECIAL_PAYLOAD;
Ming Lin8093f7c2016-04-12 13:10:14 -0600679
Christoph Hellwigfc17b652017-06-03 09:38:05 +0200680 return BLK_STS_OK;
Ming Lin8093f7c2016-04-12 13:10:14 -0600681}
Ming Lin8093f7c2016-04-12 13:10:14 -0600682
Chaitanya Kulkarni6e023182018-12-17 22:42:03 -0500683static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
684 struct request *req, struct nvme_command *cmnd)
685{
686 if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
687 return nvme_setup_discard(ns, req, cmnd);
688
689 cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
690 cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
691 cmnd->write_zeroes.slba =
Damien Le Moal314d48d2019-10-21 12:40:03 +0900692 cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
Chaitanya Kulkarni6e023182018-12-17 22:42:03 -0500693 cmnd->write_zeroes.length =
694 cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
695 cmnd->write_zeroes.control = 0;
696 return BLK_STS_OK;
697}
698
Christoph Hellwigebe6d872017-06-12 18:36:32 +0200699static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
Keith Busch240e6ee2020-06-29 12:06:41 -0700700 struct request *req, struct nvme_command *cmnd,
701 enum nvme_opcode op)
Ming Lin8093f7c2016-04-12 13:10:14 -0600702{
Jens Axboef5d11842017-06-27 12:03:06 -0600703 struct nvme_ctrl *ctrl = ns->ctrl;
Ming Lin8093f7c2016-04-12 13:10:14 -0600704 u16 control = 0;
705 u32 dsmgmt = 0;
706
707 if (req->cmd_flags & REQ_FUA)
708 control |= NVME_RW_FUA;
709 if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
710 control |= NVME_RW_LR;
711
712 if (req->cmd_flags & REQ_RAHEAD)
713 dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
714
Keith Busch240e6ee2020-06-29 12:06:41 -0700715 cmnd->rw.opcode = op;
Christoph Hellwiged754e52017-11-09 13:50:43 +0100716 cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
Damien Le Moal314d48d2019-10-21 12:40:03 +0900717 cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
Ming Lin8093f7c2016-04-12 13:10:14 -0600718 cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
719
Jens Axboef5d11842017-06-27 12:03:06 -0600720 if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams)
721 nvme_assign_write_stream(ctrl, req, &control, &dsmgmt);
722
Ming Lin8093f7c2016-04-12 13:10:14 -0600723 if (ns->ms) {
Christoph Hellwig715ea9e2017-11-07 17:27:34 +0100724 /*
725 * If formated with metadata, the block layer always provides a
726 * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled. Else
727 * we enable the PRACT bit for protection information or set the
728 * namespace capacity to zero to prevent any I/O.
729 */
730 if (!blk_integrity_rq(req)) {
731 if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
732 return BLK_STS_NOTSUPP;
733 control |= NVME_RW_PRINFO_PRACT;
734 }
735
Ming Lin8093f7c2016-04-12 13:10:14 -0600736 switch (ns->pi_type) {
737 case NVME_NS_DPS_PI_TYPE3:
738 control |= NVME_RW_PRINFO_PRCHK_GUARD;
739 break;
740 case NVME_NS_DPS_PI_TYPE1:
741 case NVME_NS_DPS_PI_TYPE2:
742 control |= NVME_RW_PRINFO_PRCHK_GUARD |
743 NVME_RW_PRINFO_PRCHK_REF;
Keith Busch240e6ee2020-06-29 12:06:41 -0700744 if (op == nvme_cmd_zone_append)
745 control |= NVME_RW_APPEND_PIREMAP;
Max Gurtovoyddd0bc72018-07-30 00:15:31 +0300746 cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
Ming Lin8093f7c2016-04-12 13:10:14 -0600747 break;
748 }
Ming Lin8093f7c2016-04-12 13:10:14 -0600749 }
750
751 cmnd->rw.control = cpu_to_le16(control);
752 cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
Christoph Hellwigebe6d872017-06-12 18:36:32 +0200753 return 0;
Ming Lin8093f7c2016-04-12 13:10:14 -0600754}
755
Max Gurtovoyf7f1fc32018-07-30 00:15:33 +0300756void nvme_cleanup_cmd(struct request *req)
757{
Max Gurtovoyf7f1fc32018-07-30 00:15:33 +0300758 if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
Jens Axboecb5b7262018-12-12 09:18:11 -0700759 struct nvme_ns *ns = req->rq_disk->private_data;
760 struct page *page = req->special_vec.bv_page;
761
762 if (page == ns->ctrl->discard_page)
763 clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
764 else
765 kfree(page_address(page) + req->special_vec.bv_offset);
Max Gurtovoyf7f1fc32018-07-30 00:15:33 +0300766 }
767}
768EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
769
Christoph Hellwigfc17b652017-06-03 09:38:05 +0200770blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
Ming Lin8093f7c2016-04-12 13:10:14 -0600771 struct nvme_command *cmd)
772{
Christoph Hellwigfc17b652017-06-03 09:38:05 +0200773 blk_status_t ret = BLK_STS_OK;
Ming Lin8093f7c2016-04-12 13:10:14 -0600774
James Smartbb06ec312018-04-12 09:16:15 -0600775 nvme_clear_nvme_request(req);
Christoph Hellwig987f6992017-04-05 19:18:08 +0200776
Chaitanya Kulkarni11902032018-10-29 16:44:18 -0700777 memset(cmd, 0, sizeof(*cmd));
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100778 switch (req_op(req)) {
779 case REQ_OP_DRV_IN:
780 case REQ_OP_DRV_OUT:
Logan Gunthorpe2bf5d3b2020-07-24 11:25:12 -0600781 nvme_setup_passthrough(req, cmd);
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100782 break;
783 case REQ_OP_FLUSH:
Ming Lin8093f7c2016-04-12 13:10:14 -0600784 nvme_setup_flush(ns, cmd);
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100785 break;
Keith Busch240e6ee2020-06-29 12:06:41 -0700786 case REQ_OP_ZONE_RESET_ALL:
787 case REQ_OP_ZONE_RESET:
788 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
789 break;
790 case REQ_OP_ZONE_OPEN:
791 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
792 break;
793 case REQ_OP_ZONE_CLOSE:
794 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
795 break;
796 case REQ_OP_ZONE_FINISH:
797 ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
798 break;
Christoph Hellwige850fd12017-04-05 19:21:13 +0200799 case REQ_OP_WRITE_ZEROES:
Chaitanya Kulkarni6e023182018-12-17 22:42:03 -0500800 ret = nvme_setup_write_zeroes(ns, req, cmd);
801 break;
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100802 case REQ_OP_DISCARD:
Ming Lin8093f7c2016-04-12 13:10:14 -0600803 ret = nvme_setup_discard(ns, req, cmd);
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100804 break;
805 case REQ_OP_READ:
Keith Busch240e6ee2020-06-29 12:06:41 -0700806 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
807 break;
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100808 case REQ_OP_WRITE:
Keith Busch240e6ee2020-06-29 12:06:41 -0700809 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
810 break;
811 case REQ_OP_ZONE_APPEND:
812 ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100813 break;
814 default:
815 WARN_ON_ONCE(1);
Christoph Hellwigfc17b652017-06-03 09:38:05 +0200816 return BLK_STS_IOERR;
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100817 }
Ming Lin8093f7c2016-04-12 13:10:14 -0600818
James Smart721b3912016-10-21 23:33:34 +0300819 cmd->common.command_id = req->tag;
Keith Busch5d87eb92018-06-29 16:50:01 -0600820 trace_nvme_setup_cmd(req, cmd);
Ming Lin8093f7c2016-04-12 13:10:14 -0600821 return ret;
822}
823EXPORT_SYMBOL_GPL(nvme_setup_cmd);
824
Sagi Grimberg6287b512018-12-14 11:06:07 -0800825static void nvme_end_sync_rq(struct request *rq, blk_status_t error)
826{
827 struct completion *waiting = rq->end_io_data;
828
829 rq->end_io_data = NULL;
830 complete(waiting);
831}
832
833static void nvme_execute_rq_polled(struct request_queue *q,
834 struct gendisk *bd_disk, struct request *rq, int at_head)
835{
836 DECLARE_COMPLETION_ONSTACK(wait);
837
838 WARN_ON_ONCE(!test_bit(QUEUE_FLAG_POLL, &q->queue_flags));
839
840 rq->cmd_flags |= REQ_HIPRI;
841 rq->end_io_data = &wait;
842 blk_execute_rq_nowait(q, bd_disk, rq, at_head, nvme_end_sync_rq);
843
844 while (!completion_done(&wait)) {
845 blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true);
846 cond_resched();
847 }
848}
849
Christoph Hellwig41609822015-11-20 09:00:02 +0100850/*
851 * Returns 0 on success. If the result is negative, it's a Linux error code;
852 * if the result is positive, it's an NVM Express status code
853 */
854int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
Christoph Hellwigd49187e2016-11-10 07:32:33 -0800855 union nvme_result *result, void *buffer, unsigned bufflen,
Bart Van Assche9a95e4e2017-11-09 10:49:59 -0800856 unsigned timeout, int qid, int at_head,
Sagi Grimberg6287b512018-12-14 11:06:07 -0800857 blk_mq_req_flags_t flags, bool poll)
Christoph Hellwig41609822015-11-20 09:00:02 +0100858{
859 struct request *req;
860 int ret;
861
Christoph Hellwigeb71f432016-06-13 16:45:23 +0200862 req = nvme_alloc_request(q, cmd, flags, qid);
Christoph Hellwig41609822015-11-20 09:00:02 +0100863 if (IS_ERR(req))
864 return PTR_ERR(req);
865
866 req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
867
Christoph Hellwig21d34712015-11-26 09:08:36 +0100868 if (buffer && bufflen) {
869 ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
870 if (ret)
871 goto out;
Christoph Hellwig41609822015-11-20 09:00:02 +0100872 }
873
Sagi Grimberg6287b512018-12-14 11:06:07 -0800874 if (poll)
875 nvme_execute_rq_polled(req->q, NULL, req, at_head);
876 else
877 blk_execute_rq(req->q, NULL, req, at_head);
Christoph Hellwigd49187e2016-11-10 07:32:33 -0800878 if (result)
879 *result = nvme_req(req)->result;
Christoph Hellwig27fa9bc2017-04-20 16:02:57 +0200880 if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
881 ret = -EINTR;
882 else
883 ret = nvme_req(req)->status;
Christoph Hellwig41609822015-11-20 09:00:02 +0100884 out:
885 blk_mq_free_request(req);
886 return ret;
887}
Christoph Hellwigeb71f432016-06-13 16:45:23 +0200888EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
Christoph Hellwig41609822015-11-20 09:00:02 +0100889
890int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
891 void *buffer, unsigned bufflen)
892{
Christoph Hellwigeb71f432016-06-13 16:45:23 +0200893 return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
Sagi Grimberg6287b512018-12-14 11:06:07 -0800894 NVME_QID_ANY, 0, 0, false);
Christoph Hellwig41609822015-11-20 09:00:02 +0100895}
Ming Lin576d55d2016-02-10 10:03:32 -0800896EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
Christoph Hellwig41609822015-11-20 09:00:02 +0100897
Christoph Hellwig1cad6562017-08-29 17:46:01 -0400898static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
899 unsigned len, u32 seed, bool write)
900{
901 struct bio_integrity_payload *bip;
902 int ret = -ENOMEM;
903 void *buf;
904
905 buf = kmalloc(len, GFP_KERNEL);
906 if (!buf)
907 goto out;
908
909 ret = -EFAULT;
910 if (write && copy_from_user(buf, ubuf, len))
911 goto out_free_meta;
912
913 bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
914 if (IS_ERR(bip)) {
915 ret = PTR_ERR(bip);
916 goto out_free_meta;
917 }
918
919 bip->bip_iter.bi_size = len;
920 bip->bip_iter.bi_sector = seed;
921 ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
922 offset_in_page(buf));
923 if (ret == len)
924 return buf;
925 ret = -ENOMEM;
926out_free_meta:
927 kfree(buf);
928out:
929 return ERR_PTR(ret);
930}
931
Logan Gunthorpedf21b6b2020-07-24 11:25:13 -0600932static u32 nvme_known_admin_effects(u8 opcode)
933{
934 switch (opcode) {
935 case nvme_admin_format_nvm:
936 return NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC |
937 NVME_CMD_EFFECTS_CSE_MASK;
938 case nvme_admin_sanitize_nvm:
939 return NVME_CMD_EFFECTS_CSE_MASK;
940 default:
941 break;
942 }
943 return 0;
944}
945
946u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
947{
948 u32 effects = 0;
949
950 if (ns) {
951 if (ns->head->effects)
952 effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
953 if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
954 dev_warn(ctrl->device,
955 "IO command:%02x has unhandled effects:%08x\n",
956 opcode, effects);
957 return 0;
958 }
959
960 if (ctrl->effects)
961 effects = le32_to_cpu(ctrl->effects->acs[opcode]);
962 effects |= nvme_known_admin_effects(opcode);
963
964 return effects;
965}
966EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
967
968static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
969 u8 opcode)
970{
971 u32 effects = nvme_command_effects(ctrl, ns, opcode);
972
973 /*
974 * For simplicity, IO to all namespaces is quiesced even if the command
975 * effects say only one namespace is affected.
976 */
977 if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
978 mutex_lock(&ctrl->scan_lock);
979 mutex_lock(&ctrl->subsys->lock);
980 nvme_mpath_start_freeze(ctrl->subsys);
981 nvme_mpath_wait_freeze(ctrl->subsys);
982 nvme_start_freeze(ctrl);
983 nvme_wait_freeze(ctrl);
984 }
985 return effects;
986}
987
988static void nvme_update_formats(struct nvme_ctrl *ctrl, u32 *effects)
989{
990 struct nvme_ns *ns;
991
992 down_read(&ctrl->namespaces_rwsem);
993 list_for_each_entry(ns, &ctrl->namespaces, list)
994 if (_nvme_revalidate_disk(ns->disk))
995 nvme_set_queue_dying(ns);
996 else if (blk_queue_is_zoned(ns->disk->queue)) {
997 /*
998 * IO commands are required to fully revalidate a zoned
999 * device. Force the command effects to trigger rescan
1000 * work so report zones can run in a context with
1001 * unfrozen IO queues.
1002 */
1003 *effects |= NVME_CMD_EFFECTS_NCC;
1004 }
1005 up_read(&ctrl->namespaces_rwsem);
1006}
1007
1008static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
1009{
1010 /*
1011 * Revalidate LBA changes prior to unfreezing. This is necessary to
1012 * prevent memory corruption if a logical block size was changed by
1013 * this command.
1014 */
1015 if (effects & NVME_CMD_EFFECTS_LBCC)
1016 nvme_update_formats(ctrl, &effects);
1017 if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
1018 nvme_unfreeze(ctrl);
1019 nvme_mpath_unfreeze(ctrl->subsys);
1020 mutex_unlock(&ctrl->subsys->lock);
1021 nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
1022 mutex_unlock(&ctrl->scan_lock);
1023 }
1024 if (effects & NVME_CMD_EFFECTS_CCC)
1025 nvme_init_identify(ctrl);
1026 if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1027 nvme_queue_scan(ctrl);
1028 flush_work(&ctrl->scan_work);
1029 }
1030}
1031
Logan Gunthorpe17365ae2020-07-24 11:25:14 -06001032void nvme_execute_passthru_rq(struct request *rq)
1033{
1034 struct nvme_command *cmd = nvme_req(rq)->cmd;
1035 struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl;
1036 struct nvme_ns *ns = rq->q->queuedata;
1037 struct gendisk *disk = ns ? ns->disk : NULL;
1038 u32 effects;
1039
1040 effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
1041 blk_execute_rq(rq->q, disk, rq, 0);
1042 nvme_passthru_end(ctrl, effects);
1043}
1044EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU);
1045
Keith Busch63263d62017-08-29 17:46:04 -04001046static int nvme_submit_user_cmd(struct request_queue *q,
Keith Busch485783c2017-08-29 17:46:03 -04001047 struct nvme_command *cmd, void __user *ubuffer,
1048 unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
Marta Rybczynska65e68ed2019-09-24 15:14:52 +02001049 u32 meta_seed, u64 *result, unsigned timeout)
Christoph Hellwig41609822015-11-20 09:00:02 +01001050{
Christoph Hellwig7a5abb42016-06-06 23:20:49 +02001051 bool write = nvme_is_write(cmd);
Keith Busch0b7f1f262015-10-23 09:47:28 -06001052 struct nvme_ns *ns = q->queuedata;
1053 struct gendisk *disk = ns ? ns->disk : NULL;
Christoph Hellwig41609822015-11-20 09:00:02 +01001054 struct request *req;
Keith Busch0b7f1f262015-10-23 09:47:28 -06001055 struct bio *bio = NULL;
1056 void *meta = NULL;
Christoph Hellwig41609822015-11-20 09:00:02 +01001057 int ret;
1058
Christoph Hellwigeb71f432016-06-13 16:45:23 +02001059 req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
Christoph Hellwig41609822015-11-20 09:00:02 +01001060 if (IS_ERR(req))
1061 return PTR_ERR(req);
1062
1063 req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
James Smartbb06ec312018-04-12 09:16:15 -06001064 nvme_req(req)->flags |= NVME_REQ_USERCMD;
Christoph Hellwig41609822015-11-20 09:00:02 +01001065
1066 if (ubuffer && bufflen) {
Christoph Hellwig21d34712015-11-26 09:08:36 +01001067 ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
1068 GFP_KERNEL);
1069 if (ret)
1070 goto out;
1071 bio = req->bio;
Christoph Hellwig74d46992017-08-23 19:10:32 +02001072 bio->bi_disk = disk;
Christoph Hellwig1cad6562017-08-29 17:46:01 -04001073 if (disk && meta_buffer && meta_len) {
1074 meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
1075 meta_seed, write);
1076 if (IS_ERR(meta)) {
1077 ret = PTR_ERR(meta);
Keith Busch0b7f1f262015-10-23 09:47:28 -06001078 goto out_unmap;
1079 }
Keith Buschf31a2112018-04-17 14:42:44 -06001080 req->cmd_flags |= REQ_INTEGRITY;
Keith Busch0b7f1f262015-10-23 09:47:28 -06001081 }
1082 }
Christoph Hellwig1cad6562017-08-29 17:46:01 -04001083
Logan Gunthorpe17365ae2020-07-24 11:25:14 -06001084 nvme_execute_passthru_rq(req);
Christoph Hellwig27fa9bc2017-04-20 16:02:57 +02001085 if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
1086 ret = -EINTR;
1087 else
1088 ret = nvme_req(req)->status;
Christoph Hellwig21d34712015-11-26 09:08:36 +01001089 if (result)
Marta Rybczynska65e68ed2019-09-24 15:14:52 +02001090 *result = le64_to_cpu(nvme_req(req)->result.u64);
Keith Busch0b7f1f262015-10-23 09:47:28 -06001091 if (meta && !ret && !write) {
1092 if (copy_to_user(meta_buffer, meta, meta_len))
1093 ret = -EFAULT;
1094 }
Keith Busch0b7f1f262015-10-23 09:47:28 -06001095 kfree(meta);
1096 out_unmap:
Christoph Hellwig74d46992017-08-23 19:10:32 +02001097 if (bio)
Keith Busch0b7f1f262015-10-23 09:47:28 -06001098 blk_rq_unmap_user(bio);
Christoph Hellwig21d34712015-11-26 09:08:36 +01001099 out:
1100 blk_mq_free_request(req);
1101 return ret;
1102}
1103
Christoph Hellwig2a842ac2017-06-03 09:38:04 +02001104static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02001105{
1106 struct nvme_ctrl *ctrl = rq->end_io_data;
James Smart86880d62018-11-27 17:04:44 -08001107 unsigned long flags;
1108 bool startka = false;
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02001109
1110 blk_mq_free_request(rq);
1111
Christoph Hellwig2a842ac2017-06-03 09:38:04 +02001112 if (status) {
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02001113 dev_err(ctrl->device,
Christoph Hellwig2a842ac2017-06-03 09:38:04 +02001114 "failed nvme_keep_alive_end_io error=%d\n",
1115 status);
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02001116 return;
1117 }
1118
Sagi Grimberg6e3ca03e2018-11-02 10:28:15 -07001119 ctrl->comp_seen = false;
James Smart86880d62018-11-27 17:04:44 -08001120 spin_lock_irqsave(&ctrl->lock, flags);
1121 if (ctrl->state == NVME_CTRL_LIVE ||
1122 ctrl->state == NVME_CTRL_CONNECTING)
1123 startka = true;
1124 spin_unlock_irqrestore(&ctrl->lock, flags);
1125 if (startka)
Nigel Kirkland97b25122020-02-10 16:01:45 -08001126 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02001127}
1128
1129static int nvme_keep_alive(struct nvme_ctrl *ctrl)
1130{
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02001131 struct request *rq;
1132
Roland Dreier0a34e462018-01-11 13:38:15 -08001133 rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd, BLK_MQ_REQ_RESERVED,
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02001134 NVME_QID_ANY);
1135 if (IS_ERR(rq))
1136 return PTR_ERR(rq);
1137
1138 rq->timeout = ctrl->kato * HZ;
1139 rq->end_io_data = ctrl;
1140
1141 blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
1142
1143 return 0;
1144}
1145
1146static void nvme_keep_alive_work(struct work_struct *work)
1147{
1148 struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1149 struct nvme_ctrl, ka_work);
Sagi Grimberg6e3ca03e2018-11-02 10:28:15 -07001150 bool comp_seen = ctrl->comp_seen;
1151
1152 if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1153 dev_dbg(ctrl->device,
1154 "reschedule traffic based keep-alive timer\n");
1155 ctrl->comp_seen = false;
Nigel Kirkland97b25122020-02-10 16:01:45 -08001156 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
Sagi Grimberg6e3ca03e2018-11-02 10:28:15 -07001157 return;
1158 }
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02001159
1160 if (nvme_keep_alive(ctrl)) {
1161 /* allocation failure, reset the controller */
1162 dev_err(ctrl->device, "keep-alive failed\n");
Christoph Hellwig39bdc592017-06-12 18:21:19 +02001163 nvme_reset_ctrl(ctrl);
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02001164 return;
1165 }
1166}
1167
Johannes Thumshirn00b683d2018-04-12 09:16:05 -06001168static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02001169{
1170 if (unlikely(ctrl->kato == 0))
1171 return;
1172
Nigel Kirkland97b25122020-02-10 16:01:45 -08001173 queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02001174}
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02001175
1176void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1177{
1178 if (unlikely(ctrl->kato == 0))
1179 return;
1180
1181 cancel_delayed_work_sync(&ctrl->ka_work);
1182}
1183EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1184
Christoph Hellwigb9a5c3d2020-04-04 10:11:28 +02001185/*
1186 * In NVMe 1.0 the CNS field was just a binary controller or namespace
1187 * flag, thus sending any new CNS opcodes has a big chance of not working.
1188 * Qemu unfortunately had that bug after reporting a 1.1 version compliance
1189 * (but not for any later version).
1190 */
1191static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl)
1192{
1193 if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)
1194 return ctrl->vs < NVME_VS(1, 2, 0);
1195 return ctrl->vs < NVME_VS(1, 1, 0);
1196}
1197
Keith Busch3f7f25a92017-06-20 15:09:56 -04001198static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
Christoph Hellwig21d34712015-11-26 09:08:36 +01001199{
1200 struct nvme_command c = { };
1201 int error;
1202
1203 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1204 c.identify.opcode = nvme_admin_identify;
Parav Pandit986994a2017-01-26 17:17:28 +02001205 c.identify.cns = NVME_ID_CNS_CTRL;
Christoph Hellwig21d34712015-11-26 09:08:36 +01001206
1207 *id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1208 if (!*id)
1209 return -ENOMEM;
1210
1211 error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1212 sizeof(struct nvme_id_ctrl));
1213 if (error)
1214 kfree(*id);
1215 return error;
1216}
1217
Niklas Cassel71010c32020-06-29 12:06:39 -07001218static bool nvme_multi_css(struct nvme_ctrl *ctrl)
1219{
1220 return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI;
1221}
1222
Chaitanya Kulkarniad95a612020-02-19 08:14:31 -08001223static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
Niklas Cassel71010c32020-06-29 12:06:39 -07001224 struct nvme_ns_id_desc *cur, bool *csi_seen)
Chaitanya Kulkarniad95a612020-02-19 08:14:31 -08001225{
1226 const char *warn_str = "ctrl returned bogus length:";
1227 void *data = cur;
1228
1229 switch (cur->nidt) {
1230 case NVME_NIDT_EUI64:
1231 if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1232 dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1233 warn_str, cur->nidl);
1234 return -1;
1235 }
1236 memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1237 return NVME_NIDT_EUI64_LEN;
1238 case NVME_NIDT_NGUID:
1239 if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1240 dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1241 warn_str, cur->nidl);
1242 return -1;
1243 }
1244 memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1245 return NVME_NIDT_NGUID_LEN;
1246 case NVME_NIDT_UUID:
1247 if (cur->nidl != NVME_NIDT_UUID_LEN) {
1248 dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1249 warn_str, cur->nidl);
1250 return -1;
1251 }
1252 uuid_copy(&ids->uuid, data + sizeof(*cur));
1253 return NVME_NIDT_UUID_LEN;
Niklas Cassel71010c32020-06-29 12:06:39 -07001254 case NVME_NIDT_CSI:
1255 if (cur->nidl != NVME_NIDT_CSI_LEN) {
1256 dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1257 warn_str, cur->nidl);
1258 return -1;
1259 }
1260 memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1261 *csi_seen = true;
1262 return NVME_NIDT_CSI_LEN;
Chaitanya Kulkarniad95a612020-02-19 08:14:31 -08001263 default:
1264 /* Skip unknown types */
1265 return cur->nidl;
1266 }
1267}
1268
Christoph Hellwigcdbff4f2017-08-16 16:14:47 +02001269static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
Christoph Hellwig002fab02017-11-09 13:50:16 +01001270 struct nvme_ns_ids *ids)
Johannes Thumshirn3b22ba22017-06-07 11:45:34 +02001271{
1272 struct nvme_command c = { };
Niklas Cassel71010c32020-06-29 12:06:39 -07001273 bool csi_seen = false;
1274 int status, pos, len;
Johannes Thumshirn3b22ba22017-06-07 11:45:34 +02001275 void *data;
Johannes Thumshirn3b22ba22017-06-07 11:45:34 +02001276
Christoph Hellwig5bedd3a2020-07-28 13:09:03 +02001277 if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1278 return 0;
1279
Johannes Thumshirn3b22ba22017-06-07 11:45:34 +02001280 c.identify.opcode = nvme_admin_identify;
1281 c.identify.nsid = cpu_to_le32(nsid);
1282 c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1283
1284 data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1285 if (!data)
1286 return -ENOMEM;
1287
Christoph Hellwigcdbff4f2017-08-16 16:14:47 +02001288 status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
Johannes Thumshirn3b22ba22017-06-07 11:45:34 +02001289 NVME_IDENTIFY_DATA_SIZE);
Christoph Hellwigfb314eb2020-03-25 14:19:35 +01001290 if (status) {
1291 dev_warn(ctrl->device,
1292 "Identify Descriptors failed (%d)\n", status);
Johannes Thumshirn3b22ba22017-06-07 11:45:34 +02001293 goto free_data;
Christoph Hellwigfb314eb2020-03-25 14:19:35 +01001294 }
Johannes Thumshirn3b22ba22017-06-07 11:45:34 +02001295
1296 for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1297 struct nvme_ns_id_desc *cur = data + pos;
1298
1299 if (cur->nidl == 0)
1300 break;
1301
Niklas Cassel71010c32020-06-29 12:06:39 -07001302 len = nvme_process_ns_desc(ctrl, ids, cur, &csi_seen);
Chaitanya Kulkarniad95a612020-02-19 08:14:31 -08001303 if (len < 0)
Niklas Cassel71010c32020-06-29 12:06:39 -07001304 break;
Johannes Thumshirn3b22ba22017-06-07 11:45:34 +02001305
1306 len += sizeof(*cur);
1307 }
Niklas Cassel71010c32020-06-29 12:06:39 -07001308
1309 if (nvme_multi_css(ctrl) && !csi_seen) {
1310 dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1311 nsid);
1312 status = -EINVAL;
1313 }
1314
Johannes Thumshirn3b22ba22017-06-07 11:45:34 +02001315free_data:
1316 kfree(data);
1317 return status;
1318}
1319
Keith Busch540c8012015-10-22 15:45:06 -06001320static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
1321{
1322 struct nvme_command c = { };
1323
1324 c.identify.opcode = nvme_admin_identify;
Parav Pandit986994a2017-01-26 17:17:28 +02001325 c.identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST;
Keith Busch540c8012015-10-22 15:45:06 -06001326 c.identify.nsid = cpu_to_le32(nsid);
Minwoo Im42595eb2018-02-08 22:56:31 +09001327 return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list,
1328 NVME_IDENTIFY_DATA_SIZE);
Keith Busch540c8012015-10-22 15:45:06 -06001329}
1330
Sagi Grimberg331813f2019-08-02 18:11:42 -07001331static int nvme_identify_ns(struct nvme_ctrl *ctrl,
1332 unsigned nsid, struct nvme_id_ns **id)
Christoph Hellwig21d34712015-11-26 09:08:36 +01001333{
1334 struct nvme_command c = { };
1335 int error;
1336
1337 /* gcc-4.4.4 (at least) has issues with initializers and anon unions */
Max Gurtovoy778f0672017-01-26 17:17:27 +02001338 c.identify.opcode = nvme_admin_identify;
1339 c.identify.nsid = cpu_to_le32(nsid);
Parav Pandit986994a2017-01-26 17:17:28 +02001340 c.identify.cns = NVME_ID_CNS_NS;
Christoph Hellwig21d34712015-11-26 09:08:36 +01001341
Sagi Grimberg331813f2019-08-02 18:11:42 -07001342 *id = kmalloc(sizeof(**id), GFP_KERNEL);
1343 if (!*id)
1344 return -ENOMEM;
Christoph Hellwig21d34712015-11-26 09:08:36 +01001345
Sagi Grimberg331813f2019-08-02 18:11:42 -07001346 error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
Christoph Hellwigcdbff4f2017-08-16 16:14:47 +02001347 if (error) {
Kenneth Heitked0de5792019-04-04 12:57:45 -06001348 dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
Sagi Grimberg331813f2019-08-02 18:11:42 -07001349 kfree(*id);
Christoph Hellwigcdbff4f2017-08-16 16:14:47 +02001350 }
1351
Sagi Grimberg331813f2019-08-02 18:11:42 -07001352 return error;
Christoph Hellwig21d34712015-11-26 09:08:36 +01001353}
1354
Keith Busch1a87ee62019-05-27 01:29:01 +09001355static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1356 unsigned int dword11, void *buffer, size_t buflen, u32 *result)
Christoph Hellwig21d34712015-11-26 09:08:36 +01001357{
Keith Busch15755852020-02-20 00:59:36 +09001358 union nvme_result res = { 0 };
Christoph Hellwig21d34712015-11-26 09:08:36 +01001359 struct nvme_command c;
Christoph Hellwig1cb3cce2016-02-29 15:59:47 +01001360 int ret;
Christoph Hellwig21d34712015-11-26 09:08:36 +01001361
1362 memset(&c, 0, sizeof(c));
Keith Busch1a87ee62019-05-27 01:29:01 +09001363 c.features.opcode = op;
Christoph Hellwig21d34712015-11-26 09:08:36 +01001364 c.features.fid = cpu_to_le32(fid);
1365 c.features.dword11 = cpu_to_le32(dword11);
1366
Christoph Hellwigd49187e2016-11-10 07:32:33 -08001367 ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
Sagi Grimberg6287b512018-12-14 11:06:07 -08001368 buffer, buflen, 0, NVME_QID_ANY, 0, 0, false);
Andy Lutomirski9b47f77a2016-08-24 03:52:12 -07001369 if (ret >= 0 && result)
Christoph Hellwigd49187e2016-11-10 07:32:33 -08001370 *result = le32_to_cpu(res.u32);
Christoph Hellwig1cb3cce2016-02-29 15:59:47 +01001371 return ret;
Christoph Hellwig21d34712015-11-26 09:08:36 +01001372}
1373
Keith Busch1a87ee62019-05-27 01:29:01 +09001374int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1375 unsigned int dword11, void *buffer, size_t buflen,
1376 u32 *result)
1377{
1378 return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1379 buflen, result);
1380}
1381EXPORT_SYMBOL_GPL(nvme_set_features);
1382
1383int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1384 unsigned int dword11, void *buffer, size_t buflen,
1385 u32 *result)
1386{
1387 return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1388 buflen, result);
1389}
1390EXPORT_SYMBOL_GPL(nvme_get_features);
1391
Christoph Hellwig9a0be7a2015-11-26 11:09:06 +01001392int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1393{
1394 u32 q_count = (*count - 1) | ((*count - 1) << 16);
1395 u32 result;
1396 int status, nr_io_queues;
1397
Andy Lutomirski1a6fe742016-09-16 11:16:10 -07001398 status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
Christoph Hellwig9a0be7a2015-11-26 11:09:06 +01001399 &result);
Christoph Hellwigf5fa90d2016-06-06 23:20:50 +02001400 if (status < 0)
Christoph Hellwig9a0be7a2015-11-26 11:09:06 +01001401 return status;
1402
Christoph Hellwigf5fa90d2016-06-06 23:20:50 +02001403 /*
1404 * Degraded controllers might return an error when setting the queue
1405 * count. We still want to be able to bring them online and offer
1406 * access to the admin queue, as that might be only way to fix them up.
1407 */
1408 if (status > 0) {
Johannes Thumshirnf0425db2017-06-09 16:17:21 +02001409 dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
Christoph Hellwigf5fa90d2016-06-06 23:20:50 +02001410 *count = 0;
1411 } else {
1412 nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1413 *count = min(*count, nr_io_queues);
1414 }
1415
Christoph Hellwig9a0be7a2015-11-26 11:09:06 +01001416 return 0;
1417}
Ming Lin576d55d2016-02-10 10:03:32 -08001418EXPORT_SYMBOL_GPL(nvme_set_queue_count);
Christoph Hellwig9a0be7a2015-11-26 11:09:06 +01001419
Hannes Reineckec0561f82018-05-22 11:09:55 +02001420#define NVME_AEN_SUPPORTED \
Sagi Grimberg85f8a432019-07-12 11:02:10 -07001421 (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1422 NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
Hannes Reineckec0561f82018-05-22 11:09:55 +02001423
1424static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1425{
Weiping Zhangfa441b72018-07-03 00:34:38 +08001426 u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
Hannes Reineckec0561f82018-05-22 11:09:55 +02001427 int status;
1428
Weiping Zhangfa441b72018-07-03 00:34:38 +08001429 if (!supported_aens)
1430 return;
1431
1432 status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1433 NULL, 0, &result);
Hannes Reineckec0561f82018-05-22 11:09:55 +02001434 if (status)
1435 dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
Weiping Zhangfa441b72018-07-03 00:34:38 +08001436 supported_aens);
Sagi Grimberg93da4022019-08-22 11:25:46 -07001437
1438 queue_work(nvme_wq, &ctrl->async_event_work);
Hannes Reineckec0561f82018-05-22 11:09:55 +02001439}
1440
Nick Bowlerc95b7082020-03-28 01:09:09 -04001441/*
1442 * Convert integer values from ioctl structures to user pointers, silently
1443 * ignoring the upper bits in the compat case to match behaviour of 32-bit
1444 * kernels.
1445 */
1446static void __user *nvme_to_user_ptr(uintptr_t ptrval)
1447{
1448 if (in_compat_syscall())
1449 ptrval = (compat_uptr_t)ptrval;
1450 return (void __user *)ptrval;
1451}
1452
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001453static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1454{
1455 struct nvme_user_io io;
1456 struct nvme_command c;
1457 unsigned length, meta_len;
1458 void __user *metadata;
1459
1460 if (copy_from_user(&io, uio, sizeof(io)))
1461 return -EFAULT;
Keith Busch63088ec2016-02-24 09:15:57 -07001462 if (io.flags)
1463 return -EINVAL;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001464
1465 switch (io.opcode) {
1466 case nvme_cmd_write:
1467 case nvme_cmd_read:
1468 case nvme_cmd_compare:
1469 break;
1470 default:
1471 return -EINVAL;
1472 }
1473
1474 length = (io.nblocks + 1) << ns->lba_shift;
1475 meta_len = (io.nblocks + 1) * ns->ms;
Nick Bowlerc95b7082020-03-28 01:09:09 -04001476 metadata = nvme_to_user_ptr(io.metadata);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001477
Max Gurtovoyffc89b1d2020-05-19 17:05:49 +03001478 if (ns->features & NVME_NS_EXT_LBAS) {
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001479 length += meta_len;
1480 meta_len = 0;
1481 } else if (meta_len) {
1482 if ((io.metadata & 3) || !io.metadata)
1483 return -EINVAL;
1484 }
1485
1486 memset(&c, 0, sizeof(c));
1487 c.rw.opcode = io.opcode;
1488 c.rw.flags = io.flags;
Christoph Hellwiged754e52017-11-09 13:50:43 +01001489 c.rw.nsid = cpu_to_le32(ns->head->ns_id);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001490 c.rw.slba = cpu_to_le64(io.slba);
1491 c.rw.length = cpu_to_le16(io.nblocks);
1492 c.rw.control = cpu_to_le16(io.control);
1493 c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1494 c.rw.reftag = cpu_to_le32(io.reftag);
1495 c.rw.apptag = cpu_to_le16(io.apptag);
1496 c.rw.appmask = cpu_to_le16(io.appmask);
1497
Keith Busch63263d62017-08-29 17:46:04 -04001498 return nvme_submit_user_cmd(ns->queue, &c,
Nick Bowlerc95b7082020-03-28 01:09:09 -04001499 nvme_to_user_ptr(io.addr), length,
Bart Van Assche202359c2018-10-10 08:08:19 -07001500 metadata, meta_len, lower_32_bits(io.slba), NULL, 0);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001501}
1502
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01001503static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001504 struct nvme_passthru_cmd __user *ucmd)
1505{
1506 struct nvme_passthru_cmd cmd;
1507 struct nvme_command c;
1508 unsigned timeout = 0;
Marta Rybczynska65e68ed2019-09-24 15:14:52 +02001509 u64 result;
1510 int status;
1511
1512 if (!capable(CAP_SYS_ADMIN))
1513 return -EACCES;
1514 if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1515 return -EFAULT;
1516 if (cmd.flags)
1517 return -EINVAL;
1518
1519 memset(&c, 0, sizeof(c));
1520 c.common.opcode = cmd.opcode;
1521 c.common.flags = cmd.flags;
1522 c.common.nsid = cpu_to_le32(cmd.nsid);
1523 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1524 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1525 c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1526 c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1527 c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1528 c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1529 c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1530 c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1531
1532 if (cmd.timeout_ms)
1533 timeout = msecs_to_jiffies(cmd.timeout_ms);
1534
Marta Rybczynska65e68ed2019-09-24 15:14:52 +02001535 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
Nick Bowlerc95b7082020-03-28 01:09:09 -04001536 nvme_to_user_ptr(cmd.addr), cmd.data_len,
1537 nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
1538 0, &result, timeout);
Marta Rybczynska65e68ed2019-09-24 15:14:52 +02001539
1540 if (status >= 0) {
1541 if (put_user(result, &ucmd->result))
1542 return -EFAULT;
1543 }
1544
1545 return status;
1546}
1547
1548static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1549 struct nvme_passthru_cmd64 __user *ucmd)
1550{
1551 struct nvme_passthru_cmd64 cmd;
1552 struct nvme_command c;
1553 unsigned timeout = 0;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001554 int status;
1555
1556 if (!capable(CAP_SYS_ADMIN))
1557 return -EACCES;
1558 if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1559 return -EFAULT;
Keith Busch63088ec2016-02-24 09:15:57 -07001560 if (cmd.flags)
1561 return -EINVAL;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001562
1563 memset(&c, 0, sizeof(c));
1564 c.common.opcode = cmd.opcode;
1565 c.common.flags = cmd.flags;
1566 c.common.nsid = cpu_to_le32(cmd.nsid);
1567 c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1568 c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
Chaitanya Kulkarnib7c8f362018-12-12 15:11:37 -08001569 c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1570 c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1571 c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1572 c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1573 c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1574 c.common.cdw15 = cpu_to_le32(cmd.cdw15);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001575
1576 if (cmd.timeout_ms)
1577 timeout = msecs_to_jiffies(cmd.timeout_ms);
1578
1579 status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
Nick Bowlerc95b7082020-03-28 01:09:09 -04001580 nvme_to_user_ptr(cmd.addr), cmd.data_len,
1581 nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
Keith Busch63263d62017-08-29 17:46:04 -04001582 0, &cmd.result, timeout);
Keith Busch84fef622017-11-07 10:28:32 -07001583
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001584 if (status >= 0) {
1585 if (put_user(cmd.result, &ucmd->result))
1586 return -EFAULT;
1587 }
1588
1589 return status;
1590}
1591
Christoph Hellwig32acab32017-11-02 12:59:30 +01001592/*
1593 * Issue ioctl requests on the first available path. Note that unlike normal
1594 * block layer requests we will not retry failed request on another controller.
1595 */
Keith Busch240e6ee2020-06-29 12:06:41 -07001596struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk,
Christoph Hellwig32acab32017-11-02 12:59:30 +01001597 struct nvme_ns_head **head, int *srcu_idx)
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001598{
Christoph Hellwig32acab32017-11-02 12:59:30 +01001599#ifdef CONFIG_NVME_MULTIPATH
1600 if (disk->fops == &nvme_ns_head_ops) {
Christoph Hellwig100c8152019-05-17 02:47:33 -07001601 struct nvme_ns *ns;
1602
Christoph Hellwig32acab32017-11-02 12:59:30 +01001603 *head = disk->private_data;
1604 *srcu_idx = srcu_read_lock(&(*head)->srcu);
Christoph Hellwig100c8152019-05-17 02:47:33 -07001605 ns = nvme_find_path(*head);
1606 if (!ns)
1607 srcu_read_unlock(&(*head)->srcu, *srcu_idx);
1608 return ns;
Christoph Hellwig32acab32017-11-02 12:59:30 +01001609 }
1610#endif
1611 *head = NULL;
1612 *srcu_idx = -1;
1613 return disk->private_data;
1614}
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001615
Keith Busch240e6ee2020-06-29 12:06:41 -07001616void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx)
Christoph Hellwig32acab32017-11-02 12:59:30 +01001617{
1618 if (head)
1619 srcu_read_unlock(&head->srcu, idx);
1620}
1621
Marta Rybczynska65e68ed2019-09-24 15:14:52 +02001622static bool is_ctrl_ioctl(unsigned int cmd)
1623{
1624 if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD)
1625 return true;
1626 if (is_sed_ioctl(cmd))
1627 return true;
1628 return false;
1629}
1630
1631static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
1632 void __user *argp,
1633 struct nvme_ns_head *head,
1634 int srcu_idx)
1635{
1636 struct nvme_ctrl *ctrl = ns->ctrl;
1637 int ret;
1638
1639 nvme_get_ctrl(ns->ctrl);
1640 nvme_put_ns_from_disk(head, srcu_idx);
1641
1642 switch (cmd) {
1643 case NVME_IOCTL_ADMIN_CMD:
1644 ret = nvme_user_cmd(ctrl, NULL, argp);
1645 break;
1646 case NVME_IOCTL_ADMIN64_CMD:
1647 ret = nvme_user_cmd64(ctrl, NULL, argp);
1648 break;
1649 default:
1650 ret = sed_ioctl(ctrl->opal_dev, cmd, argp);
1651 break;
1652 }
1653 nvme_put_ctrl(ctrl);
1654 return ret;
1655}
1656
Christoph Hellwig32acab32017-11-02 12:59:30 +01001657static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
1658 unsigned int cmd, unsigned long arg)
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001659{
Christoph Hellwig32acab32017-11-02 12:59:30 +01001660 struct nvme_ns_head *head = NULL;
Christoph Hellwig90ec6112019-05-17 02:47:35 -07001661 void __user *argp = (void __user *)arg;
Christoph Hellwig32acab32017-11-02 12:59:30 +01001662 struct nvme_ns *ns;
1663 int srcu_idx, ret;
1664
1665 ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
1666 if (unlikely(!ns))
Christoph Hellwig100c8152019-05-17 02:47:33 -07001667 return -EWOULDBLOCK;
1668
Christoph Hellwig5fb4aac2019-05-17 11:47:36 +02001669 /*
1670 * Handle ioctls that apply to the controller instead of the namespace
1671 * seperately and drop the ns SRCU reference early. This avoids a
1672 * deadlock when deleting namespaces using the passthrough interface.
1673 */
Marta Rybczynska65e68ed2019-09-24 15:14:52 +02001674 if (is_ctrl_ioctl(cmd))
1675 return nvme_handle_ctrl_ioctl(ns, cmd, argp, head, srcu_idx);
Christoph Hellwig5fb4aac2019-05-17 11:47:36 +02001676
Christoph Hellwig90ec6112019-05-17 02:47:35 -07001677 switch (cmd) {
1678 case NVME_IOCTL_ID:
1679 force_successful_syscall_return();
1680 ret = ns->head->ns_id;
1681 break;
Christoph Hellwig90ec6112019-05-17 02:47:35 -07001682 case NVME_IOCTL_IO_CMD:
1683 ret = nvme_user_cmd(ns->ctrl, ns, argp);
1684 break;
1685 case NVME_IOCTL_SUBMIT_IO:
1686 ret = nvme_submit_io(ns, argp);
1687 break;
Marta Rybczynska65e68ed2019-09-24 15:14:52 +02001688 case NVME_IOCTL_IO64_CMD:
1689 ret = nvme_user_cmd64(ns->ctrl, ns, argp);
1690 break;
Christoph Hellwig90ec6112019-05-17 02:47:35 -07001691 default:
1692 if (ns->ndev)
1693 ret = nvme_nvm_ioctl(ns, cmd, arg);
Christoph Hellwig90ec6112019-05-17 02:47:35 -07001694 else
1695 ret = -ENOTTY;
1696 }
1697
Christoph Hellwig32acab32017-11-02 12:59:30 +01001698 nvme_put_ns_from_disk(head, srcu_idx);
1699 return ret;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001700}
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001701
masahiro31.yamada@kioxia.comc225b612020-03-05 11:13:29 +00001702#ifdef CONFIG_COMPAT
1703struct nvme_user_io32 {
1704 __u8 opcode;
1705 __u8 flags;
1706 __u16 control;
1707 __u16 nblocks;
1708 __u16 rsvd;
1709 __u64 metadata;
1710 __u64 addr;
1711 __u64 slba;
1712 __u32 dsmgmt;
1713 __u32 reftag;
1714 __u16 apptag;
1715 __u16 appmask;
1716} __attribute__((__packed__));
1717
1718#define NVME_IOCTL_SUBMIT_IO32 _IOW('N', 0x42, struct nvme_user_io32)
1719
1720static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
1721 unsigned int cmd, unsigned long arg)
1722{
1723 /*
1724 * Corresponds to the difference of NVME_IOCTL_SUBMIT_IO
1725 * between 32 bit programs and 64 bit kernel.
1726 * The cause is that the results of sizeof(struct nvme_user_io),
1727 * which is used to define NVME_IOCTL_SUBMIT_IO,
1728 * are not same between 32 bit compiler and 64 bit compiler.
1729 * NVME_IOCTL_SUBMIT_IO32 is for 64 bit kernel handling
1730 * NVME_IOCTL_SUBMIT_IO issued from 32 bit programs.
1731 * Other IOCTL numbers are same between 32 bit and 64 bit.
1732 * So there is nothing to do regarding to other IOCTL numbers.
1733 */
1734 if (cmd == NVME_IOCTL_SUBMIT_IO32)
1735 return nvme_ioctl(bdev, mode, NVME_IOCTL_SUBMIT_IO, arg);
1736
1737 return nvme_ioctl(bdev, mode, cmd, arg);
1738}
1739#else
1740#define nvme_compat_ioctl NULL
1741#endif /* CONFIG_COMPAT */
1742
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001743static int nvme_open(struct block_device *bdev, fmode_t mode)
1744{
Christoph Hellwigc6424a92017-10-18 13:22:00 +02001745 struct nvme_ns *ns = bdev->bd_disk->private_data;
1746
Christoph Hellwig32acab32017-11-02 12:59:30 +01001747#ifdef CONFIG_NVME_MULTIPATH
1748 /* should never be called due to GENHD_FL_HIDDEN */
1749 if (WARN_ON_ONCE(ns->head->disk))
Nitzan Carmi85088c42018-01-04 17:56:13 +02001750 goto fail;
Christoph Hellwig32acab32017-11-02 12:59:30 +01001751#endif
Christoph Hellwigc6424a92017-10-18 13:22:00 +02001752 if (!kref_get_unless_zero(&ns->kref))
Nitzan Carmi85088c42018-01-04 17:56:13 +02001753 goto fail;
1754 if (!try_module_get(ns->ctrl->ops->module))
1755 goto fail_put_ns;
1756
Christoph Hellwigc6424a92017-10-18 13:22:00 +02001757 return 0;
Nitzan Carmi85088c42018-01-04 17:56:13 +02001758
1759fail_put_ns:
1760 nvme_put_ns(ns);
1761fail:
1762 return -ENXIO;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001763}
1764
1765static void nvme_release(struct gendisk *disk, fmode_t mode)
1766{
Nitzan Carmi85088c42018-01-04 17:56:13 +02001767 struct nvme_ns *ns = disk->private_data;
1768
1769 module_put(ns->ctrl->ops->module);
1770 nvme_put_ns(ns);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001771}
1772
1773static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1774{
1775 /* some standard values */
1776 geo->heads = 1 << 6;
1777 geo->sectors = 1 << 5;
1778 geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1779 return 0;
1780}
1781
1782#ifdef CONFIG_BLK_DEV_INTEGRITY
Max Gurtovoy95093352020-05-19 17:05:52 +03001783static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
1784 u32 max_integrity_segments)
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001785{
1786 struct blk_integrity integrity;
1787
Jay Freyenseefa9a89f2016-07-20 21:26:16 -06001788 memset(&integrity, 0, sizeof(integrity));
Christoph Hellwig39b7baa2017-11-02 21:28:53 +03001789 switch (pi_type) {
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001790 case NVME_NS_DPS_PI_TYPE3:
1791 integrity.profile = &t10_pi_type3_crc;
Nicholas Bellingerba36c212016-04-09 03:04:42 +00001792 integrity.tag_size = sizeof(u16) + sizeof(u32);
1793 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001794 break;
1795 case NVME_NS_DPS_PI_TYPE1:
1796 case NVME_NS_DPS_PI_TYPE2:
1797 integrity.profile = &t10_pi_type1_crc;
Nicholas Bellingerba36c212016-04-09 03:04:42 +00001798 integrity.tag_size = sizeof(u16);
1799 integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001800 break;
1801 default:
1802 integrity.profile = NULL;
1803 break;
1804 }
Christoph Hellwig39b7baa2017-11-02 21:28:53 +03001805 integrity.tuple_size = ms;
1806 blk_integrity_register(disk, &integrity);
Max Gurtovoy95093352020-05-19 17:05:52 +03001807 blk_queue_max_integrity_segments(disk->queue, max_integrity_segments);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001808}
1809#else
Max Gurtovoy95093352020-05-19 17:05:52 +03001810static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
1811 u32 max_integrity_segments)
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001812{
1813}
1814#endif /* CONFIG_BLK_DEV_INTEGRITY */
1815
Christoph Hellwig263185712019-03-13 18:55:07 +01001816static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001817{
Jens Axboe38317612018-05-02 11:06:54 -06001818 struct nvme_ctrl *ctrl = ns->ctrl;
Christoph Hellwig263185712019-03-13 18:55:07 +01001819 struct request_queue *queue = disk->queue;
Christoph Hellwig30e5e922017-11-02 21:28:54 +03001820 u32 size = queue_logical_block_size(queue);
1821
Jens Axboe38317612018-05-02 11:06:54 -06001822 if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) {
1823 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, queue);
1824 return;
1825 }
1826
1827 if (ctrl->nr_streams && ns->sws && ns->sgs)
1828 size *= ns->sws * ns->sgs;
Keith Busch08095e72016-03-04 13:15:17 -07001829
Christoph Hellwigb35ba012017-02-08 14:46:50 +01001830 BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1831 NVME_DSM_MAX_RANGES);
1832
David Disseldorpb224f612017-11-24 16:30:53 +01001833 queue->limits.discard_alignment = 0;
Christoph Hellwig30e5e922017-11-02 21:28:54 +03001834 queue->limits.discard_granularity = size;
Jens Axboef5d11842017-06-27 12:03:06 -06001835
Jens Axboe38317612018-05-02 11:06:54 -06001836 /* If discard is already enabled, don't reset queue limits */
1837 if (blk_queue_flag_test_and_set(QUEUE_FLAG_DISCARD, queue))
1838 return;
1839
Christoph Hellwig30e5e922017-11-02 21:28:54 +03001840 blk_queue_max_discard_sectors(queue, UINT_MAX);
1841 blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES);
Christoph Hellwige850fd12017-04-05 19:21:13 +02001842
1843 if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
Christoph Hellwig30e5e922017-11-02 21:28:54 +03001844 blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001845}
1846
Christoph Hellwig9f0916a2019-03-13 18:55:08 +01001847static void nvme_config_write_zeroes(struct gendisk *disk, struct nvme_ns *ns)
Chaitanya Kulkarni6e023182018-12-17 22:42:03 -05001848{
Damien Le Moale08f2ae2019-10-21 12:40:04 +09001849 u64 max_blocks;
Chaitanya Kulkarni6e023182018-12-17 22:42:03 -05001850
Christoph Hellwig7b210e42019-03-13 18:55:05 +01001851 if (!(ns->ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) ||
1852 (ns->ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
Chaitanya Kulkarni6e023182018-12-17 22:42:03 -05001853 return;
1854 /*
1855 * Even though NVMe spec explicitly states that MDTS is not
1856 * applicable to the write-zeroes:- "The restriction does not apply to
1857 * commands that do not transfer data between the host and the
1858 * controller (e.g., Write Uncorrectable ro Write Zeroes command).".
1859 * In order to be more cautious use controller's max_hw_sectors value
1860 * to configure the maximum sectors for the write-zeroes which is
1861 * configured based on the controller's MDTS field in the
1862 * nvme_init_identify() if available.
1863 */
1864 if (ns->ctrl->max_hw_sectors == UINT_MAX)
Damien Le Moale08f2ae2019-10-21 12:40:04 +09001865 max_blocks = (u64)USHRT_MAX + 1;
Chaitanya Kulkarni6e023182018-12-17 22:42:03 -05001866 else
Damien Le Moale08f2ae2019-10-21 12:40:04 +09001867 max_blocks = ns->ctrl->max_hw_sectors + 1;
Chaitanya Kulkarni6e023182018-12-17 22:42:03 -05001868
Damien Le Moale08f2ae2019-10-21 12:40:04 +09001869 blk_queue_max_write_zeroes_sectors(disk->queue,
1870 nvme_lba_to_sect(ns, max_blocks));
Chaitanya Kulkarni6e023182018-12-17 22:42:03 -05001871}
1872
Sagi Grimberg538af882019-08-02 18:16:12 -07001873static int nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid,
Christoph Hellwig002fab02017-11-09 13:50:16 +01001874 struct nvme_id_ns *id, struct nvme_ns_ids *ids)
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01001875{
Christoph Hellwig002fab02017-11-09 13:50:16 +01001876 memset(ids, 0, sizeof(*ids));
1877
Christoph Hellwigcdbff4f2017-08-16 16:14:47 +02001878 if (ctrl->vs >= NVME_VS(1, 1, 0))
Christoph Hellwig002fab02017-11-09 13:50:16 +01001879 memcpy(ids->eui64, id->eui64, sizeof(id->eui64));
Christoph Hellwigcdbff4f2017-08-16 16:14:47 +02001880 if (ctrl->vs >= NVME_VS(1, 2, 0))
Christoph Hellwig002fab02017-11-09 13:50:16 +01001881 memcpy(ids->nguid, id->nguid, sizeof(id->nguid));
Niklas Cassel71010c32020-06-29 12:06:39 -07001882 if (ctrl->vs >= NVME_VS(1, 3, 0) || nvme_multi_css(ctrl))
Christoph Hellwigfb314eb2020-03-25 14:19:35 +01001883 return nvme_identify_ns_descs(ctrl, nsid, ids);
1884 return 0;
Matias Bjørlingac81bfa92016-09-16 14:25:04 +02001885}
1886
Christoph Hellwiged754e52017-11-09 13:50:43 +01001887static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids)
1888{
1889 return !uuid_is_null(&ids->uuid) ||
1890 memchr_inv(ids->nguid, 0, sizeof(ids->nguid)) ||
1891 memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
1892}
1893
Christoph Hellwig002fab02017-11-09 13:50:16 +01001894static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1895{
1896 return uuid_equal(&a->uuid, &b->uuid) &&
1897 memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
Niklas Cassel71010c32020-06-29 12:06:39 -07001898 memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1899 a->csi == b->csi;
Christoph Hellwig002fab02017-11-09 13:50:16 +01001900}
1901
Keith Busch31fdad72020-04-09 09:09:08 -07001902static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1903 u32 *phys_bs, u32 *io_opt)
Keith Buschbc1af002020-04-09 09:09:07 -07001904{
1905 struct streams_directive_params s;
1906 int ret;
1907
1908 if (!ctrl->nr_streams)
1909 return 0;
1910
1911 ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id);
1912 if (ret)
1913 return ret;
1914
1915 ns->sws = le32_to_cpu(s.sws);
1916 ns->sgs = le16_to_cpu(s.sgs);
1917
1918 if (ns->sws) {
Keith Busch31fdad72020-04-09 09:09:08 -07001919 *phys_bs = ns->sws * (1 << ns->lba_shift);
Keith Buschbc1af002020-04-09 09:09:07 -07001920 if (ns->sgs)
Keith Busch31fdad72020-04-09 09:09:08 -07001921 *io_opt = *phys_bs * ns->sgs;
Keith Buschbc1af002020-04-09 09:09:07 -07001922 }
1923
1924 return 0;
1925}
1926
Christoph Hellwig24b0b582017-11-02 21:28:56 +03001927static void nvme_update_disk_info(struct gendisk *disk,
1928 struct nvme_ns *ns, struct nvme_id_ns *id)
1929{
Damien Le Moale08f2ae2019-10-21 12:40:04 +09001930 sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
Jeff Liencee160f2017-12-19 13:24:15 -06001931 unsigned short bs = 1 << ns->lba_shift;
Damien Le Moal68ab60c2020-05-14 14:56:26 +09001932 u32 atomic_bs, phys_bs, io_opt = 0;
Christoph Hellwig24b0b582017-11-02 21:28:56 +03001933
Sagi Grimberg01fa01742019-03-11 15:02:25 -07001934 if (ns->lba_shift > PAGE_SHIFT) {
1935 /* unsupported block size, set capacity to 0 later */
1936 bs = (1 << 9);
1937 }
Christoph Hellwig24b0b582017-11-02 21:28:56 +03001938 blk_mq_freeze_queue(disk->queue);
1939 blk_integrity_unregister(disk);
1940
Damien Le Moal68ab60c2020-05-14 14:56:26 +09001941 atomic_bs = phys_bs = bs;
Keith Busch31fdad72020-04-09 09:09:08 -07001942 nvme_setup_streams_ns(ns->ctrl, ns, &phys_bs, &io_opt);
Bart Van Assche81adb862019-06-28 09:53:31 -07001943 if (id->nabo == 0) {
1944 /*
1945 * Bit 1 indicates whether NAWUPF is defined for this namespace
1946 * and whether it should be used instead of AWUPF. If NAWUPF ==
1947 * 0 then AWUPF must be used instead.
1948 */
Keith Busch92decf12020-04-03 10:53:46 -07001949 if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
Bart Van Assche81adb862019-06-28 09:53:31 -07001950 atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
1951 else
1952 atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
Bart Van Assche81adb862019-06-28 09:53:31 -07001953 }
Keith Busch31fdad72020-04-09 09:09:08 -07001954
Keith Busch92decf12020-04-03 10:53:46 -07001955 if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
Bart Van Assche81adb862019-06-28 09:53:31 -07001956 /* NPWG = Namespace Preferred Write Granularity */
Keith Busch31fdad72020-04-09 09:09:08 -07001957 phys_bs = bs * (1 + le16_to_cpu(id->npwg));
Bart Van Assche81adb862019-06-28 09:53:31 -07001958 /* NOWS = Namespace Optimal Write Size */
Keith Busch31fdad72020-04-09 09:09:08 -07001959 io_opt = bs * (1 + le16_to_cpu(id->nows));
Bart Van Assche81adb862019-06-28 09:53:31 -07001960 }
1961
Jeff Liencee160f2017-12-19 13:24:15 -06001962 blk_queue_logical_block_size(disk->queue, bs);
Bart Van Assche81adb862019-06-28 09:53:31 -07001963 /*
1964 * Linux filesystems assume writing a single physical block is
1965 * an atomic operation. Hence limit the physical block size to the
1966 * value of the Atomic Write Unit Power Fail parameter.
1967 */
1968 blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
1969 blk_queue_io_min(disk->queue, phys_bs);
1970 blk_queue_io_opt(disk->queue, io_opt);
Jeff Liencee160f2017-12-19 13:24:15 -06001971
Max Gurtovoyb29f8482020-05-19 17:05:50 +03001972 /*
1973 * The block layer can't support LBA sizes larger than the page size
1974 * yet, so catch this early and don't allow block I/O.
1975 */
1976 if (ns->lba_shift > PAGE_SHIFT)
Christoph Hellwig24b0b582017-11-02 21:28:56 +03001977 capacity = 0;
Christoph Hellwig24b0b582017-11-02 21:28:56 +03001978
Max Gurtovoyb29f8482020-05-19 17:05:50 +03001979 /*
1980 * Register a metadata profile for PI, or the plain non-integrity NVMe
1981 * metadata masquerading as Type 0 if supported, otherwise reject block
1982 * I/O to namespaces with metadata except when the namespace supports
1983 * PI, as it can strip/insert in that case.
1984 */
1985 if (ns->ms) {
1986 if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
1987 (ns->features & NVME_NS_METADATA_SUPPORTED))
Max Gurtovoy95093352020-05-19 17:05:52 +03001988 nvme_init_integrity(disk, ns->ms, ns->pi_type,
1989 ns->ctrl->max_integrity_segments);
Max Gurtovoyb29f8482020-05-19 17:05:50 +03001990 else if (!nvme_ns_has_pi(ns))
1991 capacity = 0;
1992 }
1993
Balbir Singhcb224c32020-03-13 05:30:08 +00001994 set_capacity_revalidate_and_notify(disk, capacity, false);
Christoph Hellwigb1aafb32019-03-13 18:55:06 +01001995
Christoph Hellwig263185712019-03-13 18:55:07 +01001996 nvme_config_discard(disk, ns);
Christoph Hellwig9f0916a2019-03-13 18:55:08 +01001997 nvme_config_write_zeroes(disk, ns);
Chaitanya Kulkarni12934772018-08-07 23:01:06 -07001998
Keith Busch92decf12020-04-03 10:53:46 -07001999 if (id->nsattr & NVME_NS_ATTR_RO)
Chaitanya Kulkarni12934772018-08-07 23:01:06 -07002000 set_disk_ro(disk, true);
2001 else
2002 set_disk_ro(disk, false);
2003
Christoph Hellwig24b0b582017-11-02 21:28:56 +03002004 blk_mq_unfreeze_queue(disk->queue);
2005}
2006
Max Gurtovoy33cfdc22020-05-19 17:05:53 +03002007static int __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
Matias Bjørlingac81bfa92016-09-16 14:25:04 +02002008{
Keith Busch240e6ee2020-06-29 12:06:41 -07002009 unsigned lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
Matias Bjørlingac81bfa92016-09-16 14:25:04 +02002010 struct nvme_ns *ns = disk->private_data;
Max Gurtovoy33cfdc22020-05-19 17:05:53 +03002011 struct nvme_ctrl *ctrl = ns->ctrl;
Keith Busch240e6ee2020-06-29 12:06:41 -07002012 int ret;
Keith Busch38adf942020-04-09 09:09:06 -07002013 u32 iob;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002014
2015 /*
2016 * If identify namespace failed, use default 512 byte block size so
2017 * block layer can use before failing read/write for 0 capacity.
2018 */
Keith Busch240e6ee2020-06-29 12:06:41 -07002019 ns->lba_shift = id->lbaf[lbaf].ds;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002020 if (ns->lba_shift == 0)
2021 ns->lba_shift = 9;
Keith Busch38adf942020-04-09 09:09:06 -07002022
Niklas Cassel71010c32020-06-29 12:06:39 -07002023 switch (ns->head->ids.csi) {
2024 case NVME_CSI_NVM:
2025 break;
Keith Busch240e6ee2020-06-29 12:06:41 -07002026 case NVME_CSI_ZNS:
2027 ret = nvme_update_zone_info(disk, ns, lbaf);
2028 if (ret) {
2029 dev_warn(ctrl->device,
2030 "failed to add zoned namespace:%u ret:%d\n",
2031 ns->head->ns_id, ret);
2032 return ret;
2033 }
2034 break;
Niklas Cassel71010c32020-06-29 12:06:39 -07002035 default:
Keith Busch240e6ee2020-06-29 12:06:41 -07002036 dev_warn(ctrl->device, "unknown csi:%u ns:%u\n",
Niklas Cassel71010c32020-06-29 12:06:39 -07002037 ns->head->ids.csi, ns->head->ns_id);
2038 return -ENODEV;
2039 }
2040
Max Gurtovoy33cfdc22020-05-19 17:05:53 +03002041 if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
2042 is_power_of_2(ctrl->max_hw_sectors))
2043 iob = ctrl->max_hw_sectors;
Keith Busch38adf942020-04-09 09:09:06 -07002044 else
2045 iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
2046
Max Gurtovoyffc89b1d2020-05-19 17:05:49 +03002047 ns->features = 0;
Keith Busch240e6ee2020-06-29 12:06:41 -07002048 ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
Christoph Hellwigb5be3b32017-11-02 21:28:52 +03002049 /* the PI implementation requires metadata equal t10 pi tuple size */
2050 if (ns->ms == sizeof(struct t10_pi_tuple))
2051 ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
2052 else
2053 ns->pi_type = 0;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002054
Max Gurtovoyb29f8482020-05-19 17:05:50 +03002055 if (ns->ms) {
Max Gurtovoyb29f8482020-05-19 17:05:50 +03002056 /*
Max Gurtovoy33cfdc22020-05-19 17:05:53 +03002057 * For PCIe only the separate metadata pointer is supported,
2058 * as the block layer supplies metadata in a separate bio_vec
2059 * chain. For Fabrics, only metadata as part of extended data
2060 * LBA is supported on the wire per the Fabrics specification,
2061 * but the HBA/HCA will do the remapping from the separate
2062 * metadata buffers for us.
Max Gurtovoyb29f8482020-05-19 17:05:50 +03002063 */
Max Gurtovoy33cfdc22020-05-19 17:05:53 +03002064 if (id->flbas & NVME_NS_FLBAS_META_EXT) {
2065 ns->features |= NVME_NS_EXT_LBAS;
2066 if ((ctrl->ops->flags & NVME_F_FABRICS) &&
2067 (ctrl->ops->flags & NVME_F_METADATA_SUPPORTED) &&
2068 ctrl->max_integrity_segments)
2069 ns->features |= NVME_NS_METADATA_SUPPORTED;
2070 } else {
2071 if (WARN_ON_ONCE(ctrl->ops->flags & NVME_F_FABRICS))
2072 return -EINVAL;
2073 if (ctrl->ops->flags & NVME_F_METADATA_SUPPORTED)
Max Gurtovoyb29f8482020-05-19 17:05:50 +03002074 ns->features |= NVME_NS_METADATA_SUPPORTED;
2075 }
2076 }
2077
Keith Busch38adf942020-04-09 09:09:06 -07002078 if (iob)
2079 blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(iob));
Christoph Hellwig24b0b582017-11-02 21:28:56 +03002080 nvme_update_disk_info(disk, ns, id);
Christoph Hellwig32acab32017-11-02 12:59:30 +01002081#ifdef CONFIG_NVME_MULTIPATH
Sagi Grimberg8f676b852018-11-02 11:22:13 -07002082 if (ns->head->disk) {
Christoph Hellwig32acab32017-11-02 12:59:30 +01002083 nvme_update_disk_info(ns->head->disk, ns, id);
Christoph Hellwigb9b1a5d2020-07-20 08:12:51 +02002084 blk_stack_limits(&ns->head->disk->queue->limits,
2085 &ns->queue->limits, 0);
Anthony Iliopoulos05b29022020-07-14 13:11:59 +02002086 nvme_mpath_update_disk_size(ns->head->disk);
Sagi Grimberg8f676b852018-11-02 11:22:13 -07002087 }
Christoph Hellwig32acab32017-11-02 12:59:30 +01002088#endif
Max Gurtovoy33cfdc22020-05-19 17:05:53 +03002089 return 0;
Matias Bjørlingac81bfa92016-09-16 14:25:04 +02002090}
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002091
Keith Busch240e6ee2020-06-29 12:06:41 -07002092static int _nvme_revalidate_disk(struct gendisk *disk)
Matias Bjørlingac81bfa92016-09-16 14:25:04 +02002093{
2094 struct nvme_ns *ns = disk->private_data;
Christoph Hellwigcdbff4f2017-08-16 16:14:47 +02002095 struct nvme_ctrl *ctrl = ns->ctrl;
2096 struct nvme_id_ns *id;
Christoph Hellwig002fab02017-11-09 13:50:16 +01002097 struct nvme_ns_ids ids;
Christoph Hellwigcdbff4f2017-08-16 16:14:47 +02002098 int ret = 0;
Matias Bjørlingac81bfa92016-09-16 14:25:04 +02002099
2100 if (test_bit(NVME_NS_DEAD, &ns->flags)) {
2101 set_capacity(disk, 0);
2102 return -ENODEV;
2103 }
2104
Sagi Grimberg331813f2019-08-02 18:11:42 -07002105 ret = nvme_identify_ns(ctrl, ns->head->ns_id, &id);
2106 if (ret)
2107 goto out;
Matias Bjørlingac81bfa92016-09-16 14:25:04 +02002108
Christoph Hellwigcdbff4f2017-08-16 16:14:47 +02002109 if (id->ncap == 0) {
2110 ret = -ENODEV;
Sagi Grimberg331813f2019-08-02 18:11:42 -07002111 goto free_id;
Christoph Hellwigcdbff4f2017-08-16 16:14:47 +02002112 }
2113
Sagi Grimberg538af882019-08-02 18:16:12 -07002114 ret = nvme_report_ns_ids(ctrl, ns->head->ns_id, id, &ids);
2115 if (ret)
2116 goto free_id;
2117
Christoph Hellwiged754e52017-11-09 13:50:43 +01002118 if (!nvme_ns_ids_equal(&ns->head->ids, &ids)) {
Christoph Hellwig1d5df6a2017-08-17 14:10:00 +02002119 dev_err(ctrl->device,
Christoph Hellwiged754e52017-11-09 13:50:43 +01002120 "identifiers changed for nsid %d\n", ns->head->ns_id);
Christoph Hellwig1d5df6a2017-08-17 14:10:00 +02002121 ret = -ENODEV;
Keith Buschb2b2de72020-04-09 09:09:05 -07002122 goto free_id;
Christoph Hellwig1d5df6a2017-08-17 14:10:00 +02002123 }
2124
Max Gurtovoy33cfdc22020-05-19 17:05:53 +03002125 ret = __nvme_revalidate_disk(disk, id);
Sagi Grimberg331813f2019-08-02 18:11:42 -07002126free_id:
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002127 kfree(id);
Sagi Grimberg331813f2019-08-02 18:11:42 -07002128out:
Sagi Grimberg205da242019-08-30 11:00:59 -07002129 /*
2130 * Only fail the function if we got a fatal error back from the
2131 * device, otherwise ignore the error and just move on.
2132 */
2133 if (ret == -ENOMEM || (ret > 0 && !(ret & NVME_SC_DNR)))
2134 ret = 0;
2135 else if (ret > 0)
Sagi Grimberg331813f2019-08-02 18:11:42 -07002136 ret = blk_status_to_errno(nvme_error_status(ret));
Christoph Hellwigcdbff4f2017-08-16 16:14:47 +02002137 return ret;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002138}
2139
Keith Busch240e6ee2020-06-29 12:06:41 -07002140static int nvme_revalidate_disk(struct gendisk *disk)
2141{
2142 int ret;
2143
2144 ret = _nvme_revalidate_disk(disk);
2145 if (ret)
2146 return ret;
2147
2148#ifdef CONFIG_BLK_DEV_ZONED
2149 if (blk_queue_is_zoned(disk->queue)) {
2150 struct nvme_ns *ns = disk->private_data;
2151 struct nvme_ctrl *ctrl = ns->ctrl;
2152
2153 ret = blk_revalidate_disk_zones(disk, NULL);
2154 if (!ret)
2155 blk_queue_max_zone_append_sectors(disk->queue,
2156 ctrl->max_zone_append);
2157 }
2158#endif
2159 return ret;
2160}
2161
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002162static char nvme_pr_type(enum pr_type type)
2163{
2164 switch (type) {
2165 case PR_WRITE_EXCLUSIVE:
2166 return 1;
2167 case PR_EXCLUSIVE_ACCESS:
2168 return 2;
2169 case PR_WRITE_EXCLUSIVE_REG_ONLY:
2170 return 3;
2171 case PR_EXCLUSIVE_ACCESS_REG_ONLY:
2172 return 4;
2173 case PR_WRITE_EXCLUSIVE_ALL_REGS:
2174 return 5;
2175 case PR_EXCLUSIVE_ACCESS_ALL_REGS:
2176 return 6;
2177 default:
2178 return 0;
2179 }
2180};
2181
2182static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
2183 u64 key, u64 sa_key, u8 op)
2184{
Christoph Hellwig32acab32017-11-02 12:59:30 +01002185 struct nvme_ns_head *head = NULL;
2186 struct nvme_ns *ns;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002187 struct nvme_command c;
Christoph Hellwig32acab32017-11-02 12:59:30 +01002188 int srcu_idx, ret;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002189 u8 data[16] = { 0, };
2190
Keith Buschb0d61d52017-11-16 13:36:49 -07002191 ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
2192 if (unlikely(!ns))
2193 return -EWOULDBLOCK;
2194
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002195 put_unaligned_le64(key, &data[0]);
2196 put_unaligned_le64(sa_key, &data[8]);
2197
2198 memset(&c, 0, sizeof(c));
2199 c.common.opcode = op;
Keith Buschb0d61d52017-11-16 13:36:49 -07002200 c.common.nsid = cpu_to_le32(ns->head->ns_id);
Chaitanya Kulkarnib7c8f362018-12-12 15:11:37 -08002201 c.common.cdw10 = cpu_to_le32(cdw10);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002202
Keith Buschb0d61d52017-11-16 13:36:49 -07002203 ret = nvme_submit_sync_cmd(ns->queue, &c, data, 16);
Christoph Hellwig32acab32017-11-02 12:59:30 +01002204 nvme_put_ns_from_disk(head, srcu_idx);
2205 return ret;
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002206}
2207
2208static int nvme_pr_register(struct block_device *bdev, u64 old,
2209 u64 new, unsigned flags)
2210{
2211 u32 cdw10;
2212
2213 if (flags & ~PR_FL_IGNORE_KEY)
2214 return -EOPNOTSUPP;
2215
2216 cdw10 = old ? 2 : 0;
2217 cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
2218 cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
2219 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
2220}
2221
2222static int nvme_pr_reserve(struct block_device *bdev, u64 key,
2223 enum pr_type type, unsigned flags)
2224{
2225 u32 cdw10;
2226
2227 if (flags & ~PR_FL_IGNORE_KEY)
2228 return -EOPNOTSUPP;
2229
2230 cdw10 = nvme_pr_type(type) << 8;
2231 cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
2232 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
2233}
2234
2235static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
2236 enum pr_type type, bool abort)
2237{
Ivan Bornyakove9a98532018-05-23 17:56:11 +03002238 u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002239 return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
2240}
2241
2242static int nvme_pr_clear(struct block_device *bdev, u64 key)
2243{
Dan Carpenter8c0b3912015-12-09 13:24:06 +03002244 u32 cdw10 = 1 | (key ? 1 << 3 : 0);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002245 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
2246}
2247
2248static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2249{
Ivan Bornyakove9a98532018-05-23 17:56:11 +03002250 u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0);
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002251 return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
2252}
2253
2254static const struct pr_ops nvme_pr_ops = {
2255 .pr_register = nvme_pr_register,
2256 .pr_reserve = nvme_pr_reserve,
2257 .pr_release = nvme_pr_release,
2258 .pr_preempt = nvme_pr_preempt,
2259 .pr_clear = nvme_pr_clear,
2260};
2261
Scott Bauera98e58e52017-02-03 12:50:32 -07002262#ifdef CONFIG_BLK_SED_OPAL
Christoph Hellwig4f1244c2017-02-17 13:59:39 +01002263int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2264 bool send)
Scott Bauera98e58e52017-02-03 12:50:32 -07002265{
Christoph Hellwig4f1244c2017-02-17 13:59:39 +01002266 struct nvme_ctrl *ctrl = data;
Scott Bauera98e58e52017-02-03 12:50:32 -07002267 struct nvme_command cmd;
Scott Bauera98e58e52017-02-03 12:50:32 -07002268
2269 memset(&cmd, 0, sizeof(cmd));
2270 if (send)
2271 cmd.common.opcode = nvme_admin_security_send;
2272 else
2273 cmd.common.opcode = nvme_admin_security_recv;
Scott Bauera98e58e52017-02-03 12:50:32 -07002274 cmd.common.nsid = 0;
Chaitanya Kulkarnib7c8f362018-12-12 15:11:37 -08002275 cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2276 cmd.common.cdw11 = cpu_to_le32(len);
Scott Bauera98e58e52017-02-03 12:50:32 -07002277
2278 return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
Sagi Grimberg6287b512018-12-14 11:06:07 -08002279 ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0, false);
Scott Bauera98e58e52017-02-03 12:50:32 -07002280}
2281EXPORT_SYMBOL_GPL(nvme_sec_submit);
2282#endif /* CONFIG_BLK_SED_OPAL */
2283
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01002284static const struct block_device_operations nvme_fops = {
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002285 .owner = THIS_MODULE,
2286 .ioctl = nvme_ioctl,
masahiro31.yamada@kioxia.comc225b612020-03-05 11:13:29 +00002287 .compat_ioctl = nvme_compat_ioctl,
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002288 .open = nvme_open,
2289 .release = nvme_release,
2290 .getgeo = nvme_getgeo,
2291 .revalidate_disk= nvme_revalidate_disk,
Keith Busch240e6ee2020-06-29 12:06:41 -07002292 .report_zones = nvme_report_zones,
Christoph Hellwig1673f1f2015-11-26 10:54:19 +01002293 .pr_ops = &nvme_pr_ops,
2294};
2295
Christoph Hellwig32acab32017-11-02 12:59:30 +01002296#ifdef CONFIG_NVME_MULTIPATH
2297static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
2298{
2299 struct nvme_ns_head *head = bdev->bd_disk->private_data;
2300
2301 if (!kref_get_unless_zero(&head->ref))
2302 return -ENXIO;
2303 return 0;
2304}
2305
2306static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode)
2307{
2308 nvme_put_ns_head(disk->private_data);
2309}
2310
2311const struct block_device_operations nvme_ns_head_ops = {
2312 .owner = THIS_MODULE,
Christoph Hellwigc62b37d2020-07-01 10:59:43 +02002313 .submit_bio = nvme_ns_head_submit_bio,
Christoph Hellwig32acab32017-11-02 12:59:30 +01002314 .open = nvme_ns_head_open,
2315 .release = nvme_ns_head_release,
2316 .ioctl = nvme_ioctl,
masahiro31.yamada@kioxia.comc225b612020-03-05 11:13:29 +00002317 .compat_ioctl = nvme_compat_ioctl,
Christoph Hellwig32acab32017-11-02 12:59:30 +01002318 .getgeo = nvme_getgeo,
Keith Busch240e6ee2020-06-29 12:06:41 -07002319 .report_zones = nvme_report_zones,
Christoph Hellwig32acab32017-11-02 12:59:30 +01002320 .pr_ops = &nvme_pr_ops,
2321};
2322#endif /* CONFIG_NVME_MULTIPATH */
2323
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002324static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
2325{
2326 unsigned long timeout =
2327 ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
2328 u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
2329 int ret;
2330
2331 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
Keith Busch0df1e4f2016-10-11 13:31:58 -04002332 if (csts == ~0)
2333 return -ENODEV;
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002334 if ((csts & NVME_CSTS_RDY) == bit)
2335 break;
2336
Josh Triplett3e98c242020-02-28 18:52:28 -08002337 usleep_range(1000, 2000);
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002338 if (fatal_signal_pending(current))
2339 return -EINTR;
2340 if (time_after(jiffies, timeout)) {
Sagi Grimberg1b3c47c2016-02-10 08:51:15 -07002341 dev_err(ctrl->device,
Rupesh Girase94d2e702020-02-27 22:15:26 +05302342 "Device not ready; aborting %s, CSTS=0x%x\n",
2343 enabled ? "initialisation" : "reset", csts);
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002344 return -ENODEV;
2345 }
2346 }
2347
2348 return ret;
2349}
2350
2351/*
2352 * If the device has been passed off to us in an enabled state, just clear
2353 * the enabled bit. The spec says we should set the 'shutdown notification
2354 * bits', but doing so may cause the device to complete commands to the
2355 * admin queue ... and we don't know what memory that might be pointing at!
2356 */
Sagi Grimbergb5b05042019-07-22 17:06:54 -07002357int nvme_disable_ctrl(struct nvme_ctrl *ctrl)
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002358{
2359 int ret;
2360
2361 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2362 ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2363
2364 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2365 if (ret)
2366 return ret;
Guilherme G. Piccoli54adc012016-06-14 18:22:41 -03002367
Guilherme G. Piccolib5a10c52016-12-28 22:13:15 -02002368 if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
Guilherme G. Piccoli54adc012016-06-14 18:22:41 -03002369 msleep(NVME_QUIRK_DELAY_AMOUNT);
2370
Sagi Grimbergb5b05042019-07-22 17:06:54 -07002371 return nvme_wait_ready(ctrl, ctrl->cap, false);
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002372}
Ming Lin576d55d2016-02-10 10:03:32 -08002373EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002374
Sagi Grimbergc0f2f452019-07-22 17:06:53 -07002375int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002376{
Chaitanya Kulkarni6c3c05b2020-07-16 17:51:37 -07002377 unsigned dev_page_min;
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002378 int ret;
2379
Sagi Grimbergc0f2f452019-07-22 17:06:53 -07002380 ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2381 if (ret) {
2382 dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2383 return ret;
2384 }
2385 dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2386
Chaitanya Kulkarni6c3c05b2020-07-16 17:51:37 -07002387 if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
Sagi Grimberg1b3c47c2016-02-10 08:51:15 -07002388 dev_err(ctrl->device,
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002389 "Minimum device page size %u too large for host (%u)\n",
Chaitanya Kulkarni6c3c05b2020-07-16 17:51:37 -07002390 1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002391 return -ENODEV;
2392 }
2393
Niklas Cassel71010c32020-06-29 12:06:39 -07002394 if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2395 ctrl->ctrl_config = NVME_CC_CSS_CSI;
2396 else
2397 ctrl->ctrl_config = NVME_CC_CSS_NVM;
Chaitanya Kulkarni6c3c05b2020-07-16 17:51:37 -07002398 ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
Max Gurtovoy60b43f62017-08-13 19:21:07 +03002399 ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002400 ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2401 ctrl->ctrl_config |= NVME_CC_ENABLE;
2402
2403 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2404 if (ret)
2405 return ret;
Sagi Grimbergc0f2f452019-07-22 17:06:53 -07002406 return nvme_wait_ready(ctrl, ctrl->cap, true);
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002407}
Ming Lin576d55d2016-02-10 10:03:32 -08002408EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002409
2410int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
2411{
Martin K. Petersen07fbd322017-08-25 19:14:50 -04002412 unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002413 u32 csts;
2414 int ret;
2415
2416 ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2417 ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2418
2419 ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2420 if (ret)
2421 return ret;
2422
2423 while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2424 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
2425 break;
2426
2427 msleep(100);
2428 if (fatal_signal_pending(current))
2429 return -EINTR;
2430 if (time_after(jiffies, timeout)) {
Sagi Grimberg1b3c47c2016-02-10 08:51:15 -07002431 dev_err(ctrl->device,
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002432 "Device shutdown incomplete; abort shutdown\n");
2433 return -ENODEV;
2434 }
2435 }
2436
2437 return ret;
2438}
Ming Lin576d55d2016-02-10 10:03:32 -08002439EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
Christoph Hellwig5fd4ce12015-11-28 15:03:49 +01002440
Christoph Hellwigda358252016-03-02 18:07:11 +01002441static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
2442 struct request_queue *q)
2443{
Jens Axboe7c88cb02016-04-12 15:43:09 -06002444 bool vwc = false;
2445
Christoph Hellwigda358252016-03-02 18:07:11 +01002446 if (ctrl->max_hw_sectors) {
Christoph Hellwig45686b62016-03-02 18:07:12 +01002447 u32 max_segments =
Chaitanya Kulkarni6c3c05b2020-07-16 17:51:37 -07002448 (ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
Christoph Hellwig45686b62016-03-02 18:07:12 +01002449
Jens Axboe943e9422018-06-21 09:49:37 -06002450 max_segments = min_not_zero(max_segments, ctrl->max_segments);
Christoph Hellwigda358252016-03-02 18:07:11 +01002451 blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
Christoph Hellwig45686b62016-03-02 18:07:12 +01002452 blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
Christoph Hellwigda358252016-03-02 18:07:11 +01002453 }
Chaitanya Kulkarni6c3c05b2020-07-16 17:51:37 -07002454 blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);
Keith Busch3b2a1eb2020-05-20 19:22:53 -07002455 blk_queue_dma_alignment(q, 7);
Jens Axboe7c88cb02016-04-12 15:43:09 -06002456 if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
2457 vwc = true;
2458 blk_queue_write_cache(q, vwc, vwc);
Christoph Hellwigda358252016-03-02 18:07:11 +01002459}
2460
Jon Derrickdbf86b32017-08-16 09:51:29 +02002461static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2462{
2463 __le64 ts;
2464 int ret;
2465
2466 if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2467 return 0;
2468
2469 ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2470 ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2471 NULL);
2472 if (ret)
2473 dev_warn_once(ctrl->device,
2474 "could not set timestamp (%d)\n", ret);
2475 return ret;
2476}
2477
Keith Busch49cd84b2018-11-27 09:40:57 -07002478static int nvme_configure_acre(struct nvme_ctrl *ctrl)
2479{
2480 struct nvme_feat_host_behavior *host;
2481 int ret;
2482
2483 /* Don't bother enabling the feature if retry delay is not reported */
2484 if (!ctrl->crdt[0])
2485 return 0;
2486
2487 host = kzalloc(sizeof(*host), GFP_KERNEL);
2488 if (!host)
2489 return 0;
2490
2491 host->acre = NVME_ENABLE_ACRE;
2492 ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2493 host, sizeof(*host), NULL);
2494 kfree(host);
2495 return ret;
2496}
2497
Keith Busch634b8322017-08-10 11:23:31 +02002498static int nvme_configure_apst(struct nvme_ctrl *ctrl)
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002499{
2500 /*
2501 * APST (Autonomous Power State Transition) lets us program a
2502 * table of power state transitions that the controller will
2503 * perform automatically. We configure it with a simple
2504 * heuristic: we are willing to spend at most 2% of the time
2505 * transitioning between power states. Therefore, when running
2506 * in any given state, we will enter the next lower-power
Andy Lutomirski76e4ad02017-04-21 16:19:22 -07002507 * non-operational state after waiting 50 * (enlat + exlat)
Kai-Heng Fengda875912017-06-07 15:25:42 +08002508 * microseconds, as long as that state's exit latency is under
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002509 * the requested maximum latency.
2510 *
2511 * We will not autonomously enter any non-operational state for
2512 * which the total latency exceeds ps_max_latency_us. Users
2513 * can set ps_max_latency_us to zero to turn off APST.
2514 */
2515
2516 unsigned apste;
2517 struct nvme_feat_auto_pst *table;
Andy Lutomirskifb0dc392017-04-21 16:19:23 -07002518 u64 max_lat_us = 0;
2519 int max_ps = -1;
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002520 int ret;
2521
2522 /*
2523 * If APST isn't supported or if we haven't been initialized yet,
2524 * then don't do anything.
2525 */
2526 if (!ctrl->apsta)
Keith Busch634b8322017-08-10 11:23:31 +02002527 return 0;
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002528
2529 if (ctrl->npss > 31) {
2530 dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
Keith Busch634b8322017-08-10 11:23:31 +02002531 return 0;
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002532 }
2533
2534 table = kzalloc(sizeof(*table), GFP_KERNEL);
2535 if (!table)
Keith Busch634b8322017-08-10 11:23:31 +02002536 return 0;
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002537
Kai-Heng Feng76a5af82017-06-26 16:39:54 -04002538 if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002539 /* Turn off APST. */
2540 apste = 0;
Andy Lutomirskifb0dc392017-04-21 16:19:23 -07002541 dev_dbg(ctrl->device, "APST disabled\n");
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002542 } else {
2543 __le64 target = cpu_to_le64(0);
2544 int state;
2545
2546 /*
2547 * Walk through all states from lowest- to highest-power.
2548 * According to the spec, lower-numbered states use more
2549 * power. NPSS, despite the name, is the index of the
2550 * lowest-power state, not the number of states.
2551 */
2552 for (state = (int)ctrl->npss; state >= 0; state--) {
Kai-Heng Fengda875912017-06-07 15:25:42 +08002553 u64 total_latency_us, exit_latency_us, transition_ms;
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002554
2555 if (target)
2556 table->entries[state] = target;
2557
2558 /*
Andy Lutomirskiff5350a2017-04-20 13:37:55 -07002559 * Don't allow transitions to the deepest state
2560 * if it's quirked off.
2561 */
2562 if (state == ctrl->npss &&
2563 (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2564 continue;
2565
2566 /*
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002567 * Is this state a useful non-operational state for
2568 * higher-power states to autonomously transition to?
2569 */
2570 if (!(ctrl->psd[state].flags &
2571 NVME_PS_FLAGS_NON_OP_STATE))
2572 continue;
2573
Kai-Heng Fengda875912017-06-07 15:25:42 +08002574 exit_latency_us =
2575 (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2576 if (exit_latency_us > ctrl->ps_max_latency_us)
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002577 continue;
2578
Kai-Heng Fengda875912017-06-07 15:25:42 +08002579 total_latency_us =
2580 exit_latency_us +
2581 le32_to_cpu(ctrl->psd[state].entry_lat);
2582
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002583 /*
2584 * This state is good. Use it as the APST idle
2585 * target for higher power states.
2586 */
2587 transition_ms = total_latency_us + 19;
2588 do_div(transition_ms, 20);
2589 if (transition_ms > (1 << 24) - 1)
2590 transition_ms = (1 << 24) - 1;
2591
2592 target = cpu_to_le64((state << 3) |
2593 (transition_ms << 8));
Andy Lutomirskifb0dc392017-04-21 16:19:23 -07002594
2595 if (max_ps == -1)
2596 max_ps = state;
2597
2598 if (total_latency_us > max_lat_us)
2599 max_lat_us = total_latency_us;
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002600 }
2601
2602 apste = 1;
Andy Lutomirskifb0dc392017-04-21 16:19:23 -07002603
2604 if (max_ps == -1) {
2605 dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2606 } else {
2607 dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2608 max_ps, max_lat_us, (int)sizeof(*table), table);
2609 }
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002610 }
2611
2612 ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2613 table, sizeof(*table), NULL);
2614 if (ret)
2615 dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2616
2617 kfree(table);
Keith Busch634b8322017-08-10 11:23:31 +02002618 return ret;
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002619}
2620
2621static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2622{
2623 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2624 u64 latency;
2625
2626 switch (val) {
2627 case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2628 case PM_QOS_LATENCY_ANY:
2629 latency = U64_MAX;
2630 break;
2631
2632 default:
2633 latency = val;
2634 }
2635
2636 if (ctrl->ps_max_latency_us != latency) {
2637 ctrl->ps_max_latency_us = latency;
2638 nvme_configure_apst(ctrl);
2639 }
2640}
2641
Andy Lutomirskibd4da3a2017-02-22 13:32:36 -07002642struct nvme_core_quirk_entry {
2643 /*
2644 * NVMe model and firmware strings are padded with spaces. For
2645 * simplicity, strings in the quirk table are padded with NULLs
2646 * instead.
2647 */
2648 u16 vid;
2649 const char *mn;
2650 const char *fr;
2651 unsigned long quirks;
2652};
2653
2654static const struct nvme_core_quirk_entry core_quirks[] = {
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002655 {
Andy Lutomirskibe569452017-04-20 13:37:56 -07002656 /*
2657 * This Toshiba device seems to die using any APST states. See:
2658 * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2659 */
2660 .vid = 0x1179,
2661 .mn = "THNSF5256GPUK TOSHIBA",
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08002662 .quirks = NVME_QUIRK_NO_APST,
Mario Limonciellocb32de12019-08-16 15:16:19 -05002663 },
2664 {
2665 /*
2666 * This LiteON CL1-3D*-Q11 firmware version has a race
2667 * condition associated with actions related to suspend to idle
2668 * LiteON has resolved the problem in future firmware
2669 */
2670 .vid = 0x14a4,
2671 .fr = "22301111",
2672 .quirks = NVME_QUIRK_SIMPLE_SUSPEND,
Andy Lutomirskibe569452017-04-20 13:37:56 -07002673 }
Andy Lutomirskibd4da3a2017-02-22 13:32:36 -07002674};
2675
2676/* match is null-terminated but idstr is space-padded. */
2677static bool string_matches(const char *idstr, const char *match, size_t len)
2678{
2679 size_t matchlen;
2680
2681 if (!match)
2682 return true;
2683
2684 matchlen = strlen(match);
2685 WARN_ON_ONCE(matchlen > len);
2686
2687 if (memcmp(idstr, match, matchlen))
2688 return false;
2689
2690 for (; matchlen < len; matchlen++)
2691 if (idstr[matchlen] != ' ')
2692 return false;
2693
2694 return true;
2695}
2696
2697static bool quirk_matches(const struct nvme_id_ctrl *id,
2698 const struct nvme_core_quirk_entry *q)
2699{
2700 return q->vid == le16_to_cpu(id->vid) &&
2701 string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2702 string_matches(id->fr, q->fr, sizeof(id->fr));
2703}
2704
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002705static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2706 struct nvme_id_ctrl *id)
Christoph Hellwig180de0072017-06-26 12:39:02 +02002707{
2708 size_t nqnlen;
2709 int off;
2710
James Dingwall62993582019-01-08 10:20:51 -07002711 if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2712 nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2713 if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2714 strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2715 return;
2716 }
Christoph Hellwig180de0072017-06-26 12:39:02 +02002717
James Dingwall62993582019-01-08 10:20:51 -07002718 if (ctrl->vs >= NVME_VS(1, 2, 1))
2719 dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2720 }
Christoph Hellwig180de0072017-06-26 12:39:02 +02002721
2722 /* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002723 off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
Keith Busch3da584f2019-01-08 09:37:43 -07002724 "nqn.2014.08.org.nvmexpress:%04x%04x",
Christoph Hellwig180de0072017-06-26 12:39:02 +02002725 le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002726 memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
Christoph Hellwig180de0072017-06-26 12:39:02 +02002727 off += sizeof(id->sn);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002728 memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
Christoph Hellwig180de0072017-06-26 12:39:02 +02002729 off += sizeof(id->mn);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002730 memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2731}
2732
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002733static void nvme_release_subsystem(struct device *dev)
2734{
Logan Gunthorpee654dfd32019-07-18 17:53:50 -06002735 struct nvme_subsystem *subsys =
2736 container_of(dev, struct nvme_subsystem, dev);
2737
Keith Busch733e4b62019-09-05 10:33:54 -06002738 if (subsys->instance >= 0)
2739 ida_simple_remove(&nvme_instance_ida, subsys->instance);
Logan Gunthorpee654dfd32019-07-18 17:53:50 -06002740 kfree(subsys);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002741}
2742
2743static void nvme_destroy_subsystem(struct kref *ref)
2744{
2745 struct nvme_subsystem *subsys =
2746 container_of(ref, struct nvme_subsystem, ref);
2747
2748 mutex_lock(&nvme_subsystems_lock);
2749 list_del(&subsys->entry);
2750 mutex_unlock(&nvme_subsystems_lock);
2751
Christoph Hellwiged754e52017-11-09 13:50:43 +01002752 ida_destroy(&subsys->ns_ida);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002753 device_del(&subsys->dev);
2754 put_device(&subsys->dev);
2755}
2756
2757static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2758{
2759 kref_put(&subsys->ref, nvme_destroy_subsystem);
2760}
2761
2762static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2763{
2764 struct nvme_subsystem *subsys;
2765
2766 lockdep_assert_held(&nvme_subsystems_lock);
2767
James Smartc26aa572019-09-03 14:20:37 -07002768 /*
2769 * Fail matches for discovery subsystems. This results
2770 * in each discovery controller bound to a unique subsystem.
2771 * This avoids issues with validating controller values
2772 * that can only be true when there is a single unique subsystem.
2773 * There may be multiple and completely independent entities
2774 * that provide discovery controllers.
2775 */
2776 if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2777 return NULL;
2778
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002779 list_for_each_entry(subsys, &nvme_subsystems, entry) {
2780 if (strcmp(subsys->subnqn, subsysnqn))
2781 continue;
2782 if (!kref_get_unless_zero(&subsys->ref))
2783 continue;
2784 return subsys;
2785 }
2786
2787 return NULL;
2788}
2789
Hannes Reinecke1e496932017-11-10 10:58:23 +01002790#define SUBSYS_ATTR_RO(_name, _mode, _show) \
2791 struct device_attribute subsys_attr_##_name = \
2792 __ATTR(_name, _mode, _show, NULL)
2793
2794static ssize_t nvme_subsys_show_nqn(struct device *dev,
2795 struct device_attribute *attr,
2796 char *buf)
2797{
2798 struct nvme_subsystem *subsys =
2799 container_of(dev, struct nvme_subsystem, dev);
2800
2801 return snprintf(buf, PAGE_SIZE, "%s\n", subsys->subnqn);
2802}
2803static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2804
2805#define nvme_subsys_show_str_function(field) \
2806static ssize_t subsys_##field##_show(struct device *dev, \
2807 struct device_attribute *attr, char *buf) \
2808{ \
2809 struct nvme_subsystem *subsys = \
2810 container_of(dev, struct nvme_subsystem, dev); \
2811 return sprintf(buf, "%.*s\n", \
2812 (int)sizeof(subsys->field), subsys->field); \
2813} \
2814static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2815
2816nvme_subsys_show_str_function(model);
2817nvme_subsys_show_str_function(serial);
2818nvme_subsys_show_str_function(firmware_rev);
2819
2820static struct attribute *nvme_subsys_attrs[] = {
2821 &subsys_attr_model.attr,
2822 &subsys_attr_serial.attr,
2823 &subsys_attr_firmware_rev.attr,
2824 &subsys_attr_subsysnqn.attr,
Hannes Reinecke75c10e72019-02-18 11:43:26 +01002825#ifdef CONFIG_NVME_MULTIPATH
2826 &subsys_attr_iopolicy.attr,
2827#endif
Hannes Reinecke1e496932017-11-10 10:58:23 +01002828 NULL,
2829};
2830
2831static struct attribute_group nvme_subsys_attrs_group = {
2832 .attrs = nvme_subsys_attrs,
2833};
2834
2835static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2836 &nvme_subsys_attrs_group,
2837 NULL,
2838};
2839
Christoph Hellwig1b1031c2019-05-09 09:01:26 +02002840static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2841 struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
Israel Rukshinb837b282018-01-04 17:56:14 +02002842{
Christoph Hellwig1b1031c2019-05-09 09:01:26 +02002843 struct nvme_ctrl *tmp;
Israel Rukshinb837b282018-01-04 17:56:14 +02002844
Christoph Hellwig32fd90c2019-05-08 09:48:27 +02002845 lockdep_assert_held(&nvme_subsystems_lock);
2846
Christoph Hellwig1b1031c2019-05-09 09:01:26 +02002847 list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
Israel Rukshine7c43fe2020-03-10 16:39:10 +02002848 if (nvme_state_terminal(tmp))
Christoph Hellwig1b1031c2019-05-09 09:01:26 +02002849 continue;
2850
2851 if (tmp->cntlid == ctrl->cntlid) {
2852 dev_err(ctrl->device,
2853 "Duplicate cntlid %u with %s, rejecting\n",
2854 ctrl->cntlid, dev_name(tmp->device));
2855 return false;
2856 }
2857
Keith Busch92decf12020-04-03 10:53:46 -07002858 if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
Christoph Hellwig1b1031c2019-05-09 09:01:26 +02002859 (ctrl->opts && ctrl->opts->discovery_nqn))
2860 continue;
2861
2862 dev_err(ctrl->device,
2863 "Subsystem does not support multiple controllers\n");
2864 return false;
Israel Rukshinb837b282018-01-04 17:56:14 +02002865 }
Israel Rukshinb837b282018-01-04 17:56:14 +02002866
Christoph Hellwig1b1031c2019-05-09 09:01:26 +02002867 return true;
Israel Rukshinb837b282018-01-04 17:56:14 +02002868}
2869
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002870static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2871{
2872 struct nvme_subsystem *subsys, *found;
2873 int ret;
2874
2875 subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2876 if (!subsys)
2877 return -ENOMEM;
Keith Busch733e4b62019-09-05 10:33:54 -06002878
2879 subsys->instance = -1;
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002880 mutex_init(&subsys->lock);
2881 kref_init(&subsys->ref);
2882 INIT_LIST_HEAD(&subsys->ctrls);
Christoph Hellwiged754e52017-11-09 13:50:43 +01002883 INIT_LIST_HEAD(&subsys->nsheads);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002884 nvme_init_subnqn(subsys, ctrl, id);
2885 memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2886 memcpy(subsys->model, id->mn, sizeof(subsys->model));
2887 memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev));
2888 subsys->vendor_id = le16_to_cpu(id->vid);
2889 subsys->cmic = id->cmic;
Bart Van Assche81adb862019-06-28 09:53:31 -07002890 subsys->awupf = le16_to_cpu(id->awupf);
Hannes Reinecke75c10e72019-02-18 11:43:26 +01002891#ifdef CONFIG_NVME_MULTIPATH
2892 subsys->iopolicy = NVME_IOPOLICY_NUMA;
2893#endif
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002894
2895 subsys->dev.class = nvme_subsys_class;
2896 subsys->dev.release = nvme_release_subsystem;
Hannes Reinecke1e496932017-11-10 10:58:23 +01002897 subsys->dev.groups = nvme_subsys_attrs_groups;
Keith Busch733e4b62019-09-05 10:33:54 -06002898 dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002899 device_initialize(&subsys->dev);
2900
2901 mutex_lock(&nvme_subsystems_lock);
2902 found = __nvme_find_get_subsystem(subsys->subnqn);
2903 if (found) {
Logan Gunthorpee654dfd32019-07-18 17:53:50 -06002904 put_device(&subsys->dev);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002905 subsys = found;
Christoph Hellwig32fd90c2019-05-08 09:48:27 +02002906
Christoph Hellwig1b1031c2019-05-09 09:01:26 +02002907 if (!nvme_validate_cntlid(subsys, ctrl, id)) {
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002908 ret = -EINVAL;
Christoph Hellwig32fd90c2019-05-08 09:48:27 +02002909 goto out_put_subsystem;
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002910 }
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002911 } else {
2912 ret = device_add(&subsys->dev);
2913 if (ret) {
2914 dev_err(ctrl->device,
2915 "failed to register subsystem device.\n");
Logan Gunthorpe8c36e662019-07-31 17:35:34 -06002916 put_device(&subsys->dev);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002917 goto out_unlock;
2918 }
Christoph Hellwiged754e52017-11-09 13:50:43 +01002919 ida_init(&subsys->ns_ida);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002920 list_add_tail(&subsys->entry, &nvme_subsystems);
2921 }
2922
Dan Carpenterbc4f6e02019-09-23 17:18:36 +03002923 ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2924 dev_name(ctrl->device));
2925 if (ret) {
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002926 dev_err(ctrl->device,
2927 "failed to create sysfs link from subsystem.\n");
Christoph Hellwig32fd90c2019-05-08 09:48:27 +02002928 goto out_put_subsystem;
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002929 }
2930
Keith Busch733e4b62019-09-05 10:33:54 -06002931 if (!found)
2932 subsys->instance = ctrl->instance;
Christoph Hellwig32fd90c2019-05-08 09:48:27 +02002933 ctrl->subsys = subsys;
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002934 list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
Christoph Hellwig32fd90c2019-05-08 09:48:27 +02002935 mutex_unlock(&nvme_subsystems_lock);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002936 return 0;
2937
Christoph Hellwig32fd90c2019-05-08 09:48:27 +02002938out_put_subsystem:
2939 nvme_put_subsystem(subsys);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002940out_unlock:
2941 mutex_unlock(&nvme_subsystems_lock);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01002942 return ret;
Christoph Hellwig180de0072017-06-26 12:39:02 +02002943}
2944
Keith Buschbe93e872020-06-29 12:06:40 -07002945int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
Christoph Hellwig0e987192018-06-06 14:39:00 +02002946 void *log, size_t size, u64 offset)
Matias Bjørling70da6092018-02-26 13:55:40 +01002947{
2948 struct nvme_command c = { };
Keith Busch71fb90e2020-04-03 09:24:01 -07002949 u32 dwlen = nvme_bytes_to_numd(size);
Matias Bjørling70da6092018-02-26 13:55:40 +01002950
2951 c.get_log_page.opcode = nvme_admin_get_log_page;
Christoph Hellwig0e987192018-06-06 14:39:00 +02002952 c.get_log_page.nsid = cpu_to_le32(nsid);
Matias Bjørling70da6092018-02-26 13:55:40 +01002953 c.get_log_page.lid = log_page;
Christoph Hellwig0e987192018-06-06 14:39:00 +02002954 c.get_log_page.lsp = lsp;
Matias Bjørling70da6092018-02-26 13:55:40 +01002955 c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
2956 c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
Matias Bjørling7ec60742018-04-12 09:16:03 -06002957 c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
2958 c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
Keith Buschbe93e872020-06-29 12:06:40 -07002959 c.get_log_page.csi = csi;
Matias Bjørling70da6092018-02-26 13:55:40 +01002960
2961 return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
2962}
2963
Keith Buschbe93e872020-06-29 12:06:40 -07002964static struct nvme_cel *nvme_find_cel(struct nvme_ctrl *ctrl, u8 csi)
Keith Busch84fef622017-11-07 10:28:32 -07002965{
Keith Buschbe93e872020-06-29 12:06:40 -07002966 struct nvme_cel *cel, *ret = NULL;
2967
2968 spin_lock(&ctrl->lock);
2969 list_for_each_entry(cel, &ctrl->cels, entry) {
2970 if (cel->csi == csi) {
2971 ret = cel;
2972 break;
2973 }
2974 }
2975 spin_unlock(&ctrl->lock);
2976
2977 return ret;
2978}
2979
2980static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
2981 struct nvme_effects_log **log)
2982{
2983 struct nvme_cel *cel = nvme_find_cel(ctrl, csi);
Keith Busch84fef622017-11-07 10:28:32 -07002984 int ret;
2985
Keith Buschbe93e872020-06-29 12:06:40 -07002986 if (cel)
2987 goto out;
Keith Busch84fef622017-11-07 10:28:32 -07002988
Keith Buschbe93e872020-06-29 12:06:40 -07002989 cel = kzalloc(sizeof(*cel), GFP_KERNEL);
2990 if (!cel)
2991 return -ENOMEM;
Keith Busch84fef622017-11-07 10:28:32 -07002992
Keith Buschbe93e872020-06-29 12:06:40 -07002993 ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CMD_EFFECTS, 0, csi,
2994 &cel->log, sizeof(cel->log), 0);
Keith Busch84fef622017-11-07 10:28:32 -07002995 if (ret) {
Keith Buschbe93e872020-06-29 12:06:40 -07002996 kfree(cel);
2997 return ret;
Keith Busch84fef622017-11-07 10:28:32 -07002998 }
Keith Buschbe93e872020-06-29 12:06:40 -07002999
3000 cel->csi = csi;
3001
3002 spin_lock(&ctrl->lock);
3003 list_add_tail(&cel->entry, &ctrl->cels);
3004 spin_unlock(&ctrl->lock);
3005out:
3006 *log = &cel->log;
3007 return 0;
Christoph Hellwig21d34712015-11-26 09:08:36 +01003008}
3009
Christoph Hellwig7fd89302015-11-28 15:37:52 +01003010/*
3011 * Initialize the cached copies of the Identify data and various controller
3012 * register in our nvme_ctrl structure. This should be called as soon as
3013 * the admin queue is fully up and running.
3014 */
3015int nvme_init_identify(struct nvme_ctrl *ctrl)
3016{
3017 struct nvme_id_ctrl *id;
Christoph Hellwig7fd89302015-11-28 15:37:52 +01003018 int ret, page_shift;
Christoph Hellwiga229dbf2016-06-06 23:20:48 +02003019 u32 max_hw_sectors;
Kai-Heng Feng76a5af82017-06-26 16:39:54 -04003020 bool prev_apst_enabled;
Christoph Hellwig7fd89302015-11-28 15:37:52 +01003021
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003022 ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
3023 if (ret) {
Sagi Grimberg1b3c47c2016-02-10 08:51:15 -07003024 dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003025 return ret;
3026 }
Sagi Grimberg4fba4452019-07-22 17:06:52 -07003027 page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12;
Chaitanya Kulkarnid4047cf2020-06-01 19:41:12 -07003028 ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
Christoph Hellwig7fd89302015-11-28 15:37:52 +01003029
Gabriel Krisman Bertazi8ef20742016-10-19 09:51:05 -06003030 if (ctrl->vs >= NVME_VS(1, 1, 0))
Sagi Grimberg4fba4452019-07-22 17:06:52 -07003031 ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003032
Christoph Hellwig7fd89302015-11-28 15:37:52 +01003033 ret = nvme_identify_ctrl(ctrl, &id);
3034 if (ret) {
Sagi Grimberg1b3c47c2016-02-10 08:51:15 -07003035 dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
Christoph Hellwig7fd89302015-11-28 15:37:52 +01003036 return -EIO;
3037 }
3038
Keith Busch84fef622017-11-07 10:28:32 -07003039 if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
Keith Buschbe93e872020-06-29 12:06:40 -07003040 ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
Keith Busch84fef622017-11-07 10:28:32 -07003041 if (ret < 0)
Hannes Reinecke75c8b192018-05-25 11:06:27 +02003042 goto out_free;
Keith Busch84fef622017-11-07 10:28:32 -07003043 }
Christoph Hellwig180de0072017-06-26 12:39:02 +02003044
Guilherme G. Piccolia89fcca2019-08-14 11:26:10 -03003045 if (!(ctrl->ops->flags & NVME_F_FABRICS))
3046 ctrl->cntlid = le16_to_cpu(id->cntlid);
3047
Andy Lutomirskibd4da3a2017-02-22 13:32:36 -07003048 if (!ctrl->identified) {
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01003049 int i;
3050
3051 ret = nvme_init_subsystem(ctrl, id);
3052 if (ret)
3053 goto out_free;
3054
Andy Lutomirskibd4da3a2017-02-22 13:32:36 -07003055 /*
3056 * Check for quirks. Quirk can depend on firmware version,
3057 * so, in principle, the set of quirks present can change
3058 * across a reset. As a possible future enhancement, we
3059 * could re-scan for quirks every time we reinitialize
3060 * the device, but we'd have to make sure that the driver
3061 * behaves intelligently if the quirks change.
3062 */
Andy Lutomirskibd4da3a2017-02-22 13:32:36 -07003063 for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
3064 if (quirk_matches(id, &core_quirks[i]))
3065 ctrl->quirks |= core_quirks[i].quirks;
3066 }
3067 }
3068
Andy Lutomirskic35e30b2017-04-21 16:19:24 -07003069 if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
Johannes Thumshirnf0425db2017-06-09 16:17:21 +02003070 dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
Andy Lutomirskic35e30b2017-04-21 16:19:24 -07003071 ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
3072 }
3073
Keith Busch49cd84b2018-11-27 09:40:57 -07003074 ctrl->crdt[0] = le16_to_cpu(id->crdt1);
3075 ctrl->crdt[1] = le16_to_cpu(id->crdt2);
3076 ctrl->crdt[2] = le16_to_cpu(id->crdt3);
3077
Scott Bauer8a9ae522017-02-17 13:59:40 +01003078 ctrl->oacs = le16_to_cpu(id->oacs);
Max Gurtovoy43e2d082019-02-25 13:00:04 +02003079 ctrl->oncs = le16_to_cpu(id->oncs);
Laine Walker-Avina2d466c72019-05-20 10:13:04 -07003080 ctrl->mtfa = le16_to_cpu(id->mtfa);
Hannes Reineckec0561f82018-05-22 11:09:55 +02003081 ctrl->oaes = le32_to_cpu(id->oaes);
Guenter Roeck400b6a72019-11-06 06:35:18 -08003082 ctrl->wctemp = le16_to_cpu(id->wctemp);
3083 ctrl->cctemp = le16_to_cpu(id->cctemp);
3084
Christoph Hellwig6bf25d12015-11-20 09:36:44 +01003085 atomic_set(&ctrl->abort_limit, id->acl + 1);
Christoph Hellwig7fd89302015-11-28 15:37:52 +01003086 ctrl->vwc = id->vwc;
Christoph Hellwig7fd89302015-11-28 15:37:52 +01003087 if (id->mdts)
Christoph Hellwiga229dbf2016-06-06 23:20:48 +02003088 max_hw_sectors = 1 << (id->mdts + page_shift - 9);
Christoph Hellwig7fd89302015-11-28 15:37:52 +01003089 else
Christoph Hellwiga229dbf2016-06-06 23:20:48 +02003090 max_hw_sectors = UINT_MAX;
3091 ctrl->max_hw_sectors =
3092 min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
Christoph Hellwig7fd89302015-11-28 15:37:52 +01003093
Christoph Hellwigda358252016-03-02 18:07:11 +01003094 nvme_set_queue_limits(ctrl, ctrl->admin_q);
Christoph Hellwig07bfcd02016-06-13 16:45:26 +02003095 ctrl->sgls = le32_to_cpu(id->sgls);
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02003096 ctrl->kas = le16_to_cpu(id->kas);
Christoph Hellwig0d0b6602018-05-14 08:48:54 +02003097 ctrl->max_namespaces = le32_to_cpu(id->mnan);
Sagi Grimberg3e53ba382018-11-02 10:28:14 -07003098 ctrl->ctratt = le32_to_cpu(id->ctratt);
Christoph Hellwig07bfcd02016-06-13 16:45:26 +02003099
Martin K. Petersen07fbd322017-08-25 19:14:50 -04003100 if (id->rtd3e) {
3101 /* us -> s */
Baolin Wangf5af5772020-06-24 14:49:58 +08003102 u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
Martin K. Petersen07fbd322017-08-25 19:14:50 -04003103
3104 ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
3105 shutdown_timeout, 60);
3106
3107 if (ctrl->shutdown_timeout != shutdown_timeout)
Max Gurtovoy1a3838d2017-12-31 15:33:27 +02003108 dev_info(ctrl->device,
Martin K. Petersen07fbd322017-08-25 19:14:50 -04003109 "Shutdown timeout set to %u seconds\n",
3110 ctrl->shutdown_timeout);
3111 } else
3112 ctrl->shutdown_timeout = shutdown_timeout;
3113
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08003114 ctrl->npss = id->npss;
Kai-Heng Feng76a5af82017-06-26 16:39:54 -04003115 ctrl->apsta = id->apsta;
3116 prev_apst_enabled = ctrl->apst_enabled;
Andy Lutomirskic35e30b2017-04-21 16:19:24 -07003117 if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3118 if (force_apst && id->apsta) {
Johannes Thumshirnf0425db2017-06-09 16:17:21 +02003119 dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
Kai-Heng Feng76a5af82017-06-26 16:39:54 -04003120 ctrl->apst_enabled = true;
Andy Lutomirskic35e30b2017-04-21 16:19:24 -07003121 } else {
Kai-Heng Feng76a5af82017-06-26 16:39:54 -04003122 ctrl->apst_enabled = false;
Andy Lutomirskic35e30b2017-04-21 16:19:24 -07003123 }
3124 } else {
Kai-Heng Feng76a5af82017-06-26 16:39:54 -04003125 ctrl->apst_enabled = id->apsta;
Andy Lutomirskic35e30b2017-04-21 16:19:24 -07003126 }
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08003127 memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3128
Christoph Hellwigd3d5b872017-05-20 15:14:44 +02003129 if (ctrl->ops->flags & NVME_F_FABRICS) {
Christoph Hellwig07bfcd02016-06-13 16:45:26 +02003130 ctrl->icdoff = le16_to_cpu(id->icdoff);
3131 ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3132 ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3133 ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3134
3135 /*
3136 * In fabrics we need to verify the cntlid matches the
3137 * admin connect
3138 */
Keith Busch634b8322017-08-10 11:23:31 +02003139 if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
James Smarta8157ff2019-11-21 09:58:10 -08003140 dev_err(ctrl->device,
3141 "Mismatching cntlid: Connect %u vs Identify "
3142 "%u, rejecting\n",
3143 ctrl->cntlid, le16_to_cpu(id->cntlid));
Christoph Hellwig07bfcd02016-06-13 16:45:26 +02003144 ret = -EINVAL;
Keith Busch634b8322017-08-10 11:23:31 +02003145 goto out_free;
3146 }
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02003147
3148 if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
Johannes Thumshirnf0425db2017-06-09 16:17:21 +02003149 dev_err(ctrl->device,
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02003150 "keep-alive support is mandatory for fabrics\n");
3151 ret = -EINVAL;
Keith Busch634b8322017-08-10 11:23:31 +02003152 goto out_free;
Sagi Grimberg038bd4c2016-06-13 16:45:28 +02003153 }
Christoph Hellwig07bfcd02016-06-13 16:45:26 +02003154 } else {
Christoph Hellwigfe6d53c2017-05-12 17:16:10 +02003155 ctrl->hmpre = le32_to_cpu(id->hmpre);
3156 ctrl->hmmin = le32_to_cpu(id->hmmin);
Christoph Hellwig044a9df2017-09-11 12:09:28 -04003157 ctrl->hmminds = le32_to_cpu(id->hmminds);
3158 ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
Christoph Hellwig07bfcd02016-06-13 16:45:26 +02003159 }
Christoph Hellwigda358252016-03-02 18:07:11 +01003160
Christoph Hellwig0d0b6602018-05-14 08:48:54 +02003161 ret = nvme_mpath_init(ctrl, id);
Christoph Hellwig7fd89302015-11-28 15:37:52 +01003162 kfree(id);
Andy Lutomirskibd4da3a2017-02-22 13:32:36 -07003163
Christoph Hellwig0d0b6602018-05-14 08:48:54 +02003164 if (ret < 0)
3165 return ret;
3166
Kai-Heng Feng76a5af82017-06-26 16:39:54 -04003167 if (ctrl->apst_enabled && !prev_apst_enabled)
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08003168 dev_pm_qos_expose_latency_tolerance(ctrl->device);
Kai-Heng Feng76a5af82017-06-26 16:39:54 -04003169 else if (!ctrl->apst_enabled && prev_apst_enabled)
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08003170 dev_pm_qos_hide_latency_tolerance(ctrl->device);
3171
Keith Busch634b8322017-08-10 11:23:31 +02003172 ret = nvme_configure_apst(ctrl);
3173 if (ret < 0)
3174 return ret;
Jon Derrickdbf86b32017-08-16 09:51:29 +02003175
3176 ret = nvme_configure_timestamp(ctrl);
3177 if (ret < 0)
3178 return ret;
Keith Busch634b8322017-08-10 11:23:31 +02003179
3180 ret = nvme_configure_directives(ctrl);
3181 if (ret < 0)
3182 return ret;
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08003183
Keith Busch49cd84b2018-11-27 09:40:57 -07003184 ret = nvme_configure_acre(ctrl);
3185 if (ret < 0)
3186 return ret;
3187
Guenter Roeck400b6a72019-11-06 06:35:18 -08003188 if (!ctrl->identified)
3189 nvme_hwmon_init(ctrl);
3190
Andy Lutomirskibd4da3a2017-02-22 13:32:36 -07003191 ctrl->identified = true;
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08003192
Keith Busch634b8322017-08-10 11:23:31 +02003193 return 0;
3194
3195out_free:
3196 kfree(id);
Christoph Hellwig07bfcd02016-06-13 16:45:26 +02003197 return ret;
Christoph Hellwig7fd89302015-11-28 15:37:52 +01003198}
Ming Lin576d55d2016-02-10 10:03:32 -08003199EXPORT_SYMBOL_GPL(nvme_init_identify);
Christoph Hellwig7fd89302015-11-28 15:37:52 +01003200
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003201static int nvme_dev_open(struct inode *inode, struct file *file)
Christoph Hellwig21d34712015-11-26 09:08:36 +01003202{
Christoph Hellwiga6a51492017-10-18 16:59:25 +02003203 struct nvme_ctrl *ctrl =
3204 container_of(inode->i_cdev, struct nvme_ctrl, cdev);
Christoph Hellwig21d34712015-11-26 09:08:36 +01003205
Jianchao Wang2b1b7e72018-01-06 08:01:58 +08003206 switch (ctrl->state) {
3207 case NVME_CTRL_LIVE:
Jianchao Wang2b1b7e72018-01-06 08:01:58 +08003208 break;
3209 default:
Christoph Hellwiga6a51492017-10-18 16:59:25 +02003210 return -EWOULDBLOCK;
Jianchao Wang2b1b7e72018-01-06 08:01:58 +08003211 }
3212
Christoph Hellwiga6a51492017-10-18 16:59:25 +02003213 file->private_data = ctrl;
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003214 return 0;
Christoph Hellwig21d34712015-11-26 09:08:36 +01003215}
3216
Christoph Hellwigbfd89472015-12-24 15:27:01 +01003217static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
3218{
3219 struct nvme_ns *ns;
3220 int ret;
3221
Jianchao Wang765cc0312018-02-12 20:54:46 +08003222 down_read(&ctrl->namespaces_rwsem);
Christoph Hellwigbfd89472015-12-24 15:27:01 +01003223 if (list_empty(&ctrl->namespaces)) {
3224 ret = -ENOTTY;
3225 goto out_unlock;
3226 }
3227
3228 ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
3229 if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
Sagi Grimberg1b3c47c2016-02-10 08:51:15 -07003230 dev_warn(ctrl->device,
Christoph Hellwigbfd89472015-12-24 15:27:01 +01003231 "NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
3232 ret = -EINVAL;
3233 goto out_unlock;
3234 }
3235
Sagi Grimberg1b3c47c2016-02-10 08:51:15 -07003236 dev_warn(ctrl->device,
Christoph Hellwigbfd89472015-12-24 15:27:01 +01003237 "using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
3238 kref_get(&ns->kref);
Jianchao Wang765cc0312018-02-12 20:54:46 +08003239 up_read(&ctrl->namespaces_rwsem);
Christoph Hellwigbfd89472015-12-24 15:27:01 +01003240
3241 ret = nvme_user_cmd(ctrl, ns, argp);
3242 nvme_put_ns(ns);
3243 return ret;
3244
3245out_unlock:
Jianchao Wang765cc0312018-02-12 20:54:46 +08003246 up_read(&ctrl->namespaces_rwsem);
Christoph Hellwigbfd89472015-12-24 15:27:01 +01003247 return ret;
3248}
3249
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003250static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
3251 unsigned long arg)
3252{
3253 struct nvme_ctrl *ctrl = file->private_data;
3254 void __user *argp = (void __user *)arg;
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003255
3256 switch (cmd) {
3257 case NVME_IOCTL_ADMIN_CMD:
3258 return nvme_user_cmd(ctrl, NULL, argp);
Marta Rybczynska65e68ed2019-09-24 15:14:52 +02003259 case NVME_IOCTL_ADMIN64_CMD:
3260 return nvme_user_cmd64(ctrl, NULL, argp);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003261 case NVME_IOCTL_IO_CMD:
Christoph Hellwigbfd89472015-12-24 15:27:01 +01003262 return nvme_dev_user_cmd(ctrl, argp);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003263 case NVME_IOCTL_RESET:
Sagi Grimberg1b3c47c2016-02-10 08:51:15 -07003264 dev_warn(ctrl->device, "resetting controller\n");
Christoph Hellwigd86c4d82017-06-15 15:41:08 +02003265 return nvme_reset_ctrl_sync(ctrl);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003266 case NVME_IOCTL_SUBSYS_RESET:
3267 return nvme_reset_subsystem(ctrl);
Keith Busch9ec3bb22016-04-29 15:45:18 -06003268 case NVME_IOCTL_RESCAN:
3269 nvme_queue_scan(ctrl);
3270 return 0;
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003271 default:
3272 return -ENOTTY;
3273 }
3274}
3275
3276static const struct file_operations nvme_dev_fops = {
3277 .owner = THIS_MODULE,
3278 .open = nvme_dev_open,
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003279 .unlocked_ioctl = nvme_dev_ioctl,
Arnd Bergmann1832f2d2018-09-11 21:59:08 +02003280 .compat_ioctl = compat_ptr_ioctl,
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003281};
3282
3283static ssize_t nvme_sysfs_reset(struct device *dev,
3284 struct device_attribute *attr, const char *buf,
3285 size_t count)
3286{
3287 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3288 int ret;
3289
Christoph Hellwigd86c4d82017-06-15 15:41:08 +02003290 ret = nvme_reset_ctrl_sync(ctrl);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01003291 if (ret < 0)
3292 return ret;
3293 return count;
3294}
3295static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
3296
Keith Busch9ec3bb22016-04-29 15:45:18 -06003297static ssize_t nvme_sysfs_rescan(struct device *dev,
3298 struct device_attribute *attr, const char *buf,
3299 size_t count)
3300{
3301 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3302
3303 nvme_queue_scan(ctrl);
3304 return count;
3305}
3306static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3307
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003308static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
Keith Busch118472a2016-02-18 09:57:48 -07003309{
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003310 struct gendisk *disk = dev_to_disk(dev);
Keith Busch118472a2016-02-18 09:57:48 -07003311
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003312 if (disk->fops == &nvme_fops)
3313 return nvme_get_ns_from_dev(dev)->head;
3314 else
3315 return disk->private_data;
3316}
Johannes Thumshirn6484f5d2017-07-12 15:38:56 +02003317
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003318static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
3319 char *buf)
3320{
3321 struct nvme_ns_head *head = dev_to_ns_head(dev);
3322 struct nvme_ns_ids *ids = &head->ids;
3323 struct nvme_subsystem *subsys = head->subsys;
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01003324 int serial_len = sizeof(subsys->serial);
3325 int model_len = sizeof(subsys->model);
Keith Busch118472a2016-02-18 09:57:48 -07003326
Christoph Hellwig002fab02017-11-09 13:50:16 +01003327 if (!uuid_is_null(&ids->uuid))
3328 return sprintf(buf, "uuid.%pU\n", &ids->uuid);
Keith Busch118472a2016-02-18 09:57:48 -07003329
Christoph Hellwig002fab02017-11-09 13:50:16 +01003330 if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3331 return sprintf(buf, "eui.%16phN\n", ids->nguid);
Keith Busch118472a2016-02-18 09:57:48 -07003332
Christoph Hellwig002fab02017-11-09 13:50:16 +01003333 if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3334 return sprintf(buf, "eui.%8phN\n", ids->eui64);
Keith Busch118472a2016-02-18 09:57:48 -07003335
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01003336 while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3337 subsys->serial[serial_len - 1] == '\0'))
Keith Busch118472a2016-02-18 09:57:48 -07003338 serial_len--;
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01003339 while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3340 subsys->model[model_len - 1] == '\0'))
Keith Busch118472a2016-02-18 09:57:48 -07003341 model_len--;
3342
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01003343 return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
3344 serial_len, subsys->serial, model_len, subsys->model,
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003345 head->ns_id);
Keith Busch118472a2016-02-18 09:57:48 -07003346}
Joe Perchesc828a892017-12-19 10:15:08 -08003347static DEVICE_ATTR_RO(wwid);
Keith Busch118472a2016-02-18 09:57:48 -07003348
Johannes Thumshirnd934f982017-06-07 11:45:35 +02003349static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003350 char *buf)
Johannes Thumshirnd934f982017-06-07 11:45:35 +02003351{
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003352 return sprintf(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
Johannes Thumshirnd934f982017-06-07 11:45:35 +02003353}
Joe Perchesc828a892017-12-19 10:15:08 -08003354static DEVICE_ATTR_RO(nguid);
Johannes Thumshirnd934f982017-06-07 11:45:35 +02003355
Keith Busch2b9b6e82015-12-22 10:10:45 -07003356static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003357 char *buf)
Keith Busch2b9b6e82015-12-22 10:10:45 -07003358{
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003359 struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
Johannes Thumshirnd934f982017-06-07 11:45:35 +02003360
3361 /* For backward compatibility expose the NGUID to userspace if
3362 * we have no UUID set
3363 */
Christoph Hellwig002fab02017-11-09 13:50:16 +01003364 if (uuid_is_null(&ids->uuid)) {
Johannes Thumshirnd934f982017-06-07 11:45:35 +02003365 printk_ratelimited(KERN_WARNING
3366 "No UUID available providing old NGUID\n");
Christoph Hellwig002fab02017-11-09 13:50:16 +01003367 return sprintf(buf, "%pU\n", ids->nguid);
Johannes Thumshirnd934f982017-06-07 11:45:35 +02003368 }
Christoph Hellwig002fab02017-11-09 13:50:16 +01003369 return sprintf(buf, "%pU\n", &ids->uuid);
Keith Busch2b9b6e82015-12-22 10:10:45 -07003370}
Joe Perchesc828a892017-12-19 10:15:08 -08003371static DEVICE_ATTR_RO(uuid);
Keith Busch2b9b6e82015-12-22 10:10:45 -07003372
3373static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003374 char *buf)
Keith Busch2b9b6e82015-12-22 10:10:45 -07003375{
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003376 return sprintf(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
Keith Busch2b9b6e82015-12-22 10:10:45 -07003377}
Joe Perchesc828a892017-12-19 10:15:08 -08003378static DEVICE_ATTR_RO(eui);
Keith Busch2b9b6e82015-12-22 10:10:45 -07003379
3380static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003381 char *buf)
Keith Busch2b9b6e82015-12-22 10:10:45 -07003382{
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003383 return sprintf(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
Keith Busch2b9b6e82015-12-22 10:10:45 -07003384}
Joe Perchesc828a892017-12-19 10:15:08 -08003385static DEVICE_ATTR_RO(nsid);
Keith Busch2b9b6e82015-12-22 10:10:45 -07003386
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003387static struct attribute *nvme_ns_id_attrs[] = {
Keith Busch118472a2016-02-18 09:57:48 -07003388 &dev_attr_wwid.attr,
Keith Busch2b9b6e82015-12-22 10:10:45 -07003389 &dev_attr_uuid.attr,
Johannes Thumshirnd934f982017-06-07 11:45:35 +02003390 &dev_attr_nguid.attr,
Keith Busch2b9b6e82015-12-22 10:10:45 -07003391 &dev_attr_eui.attr,
3392 &dev_attr_nsid.attr,
Christoph Hellwig0d0b6602018-05-14 08:48:54 +02003393#ifdef CONFIG_NVME_MULTIPATH
3394 &dev_attr_ana_grpid.attr,
3395 &dev_attr_ana_state.attr,
3396#endif
Keith Busch2b9b6e82015-12-22 10:10:45 -07003397 NULL,
3398};
3399
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003400static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
Keith Busch2b9b6e82015-12-22 10:10:45 -07003401 struct attribute *a, int n)
3402{
3403 struct device *dev = container_of(kobj, struct device, kobj);
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003404 struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
Keith Busch2b9b6e82015-12-22 10:10:45 -07003405
3406 if (a == &dev_attr_uuid.attr) {
Martin Wilcka04b5de2017-09-28 21:33:23 +02003407 if (uuid_is_null(&ids->uuid) &&
Christoph Hellwig002fab02017-11-09 13:50:16 +01003408 !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
Johannes Thumshirnd934f982017-06-07 11:45:35 +02003409 return 0;
3410 }
3411 if (a == &dev_attr_nguid.attr) {
Christoph Hellwig002fab02017-11-09 13:50:16 +01003412 if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
Keith Busch2b9b6e82015-12-22 10:10:45 -07003413 return 0;
3414 }
3415 if (a == &dev_attr_eui.attr) {
Christoph Hellwig002fab02017-11-09 13:50:16 +01003416 if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
Keith Busch2b9b6e82015-12-22 10:10:45 -07003417 return 0;
3418 }
Christoph Hellwig0d0b6602018-05-14 08:48:54 +02003419#ifdef CONFIG_NVME_MULTIPATH
3420 if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
3421 if (dev_to_disk(dev)->fops != &nvme_fops) /* per-path attr */
3422 return 0;
3423 if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3424 return 0;
3425 }
3426#endif
Keith Busch2b9b6e82015-12-22 10:10:45 -07003427 return a->mode;
3428}
3429
Bart Van Asscheeb090c42018-10-08 14:28:39 -07003430static const struct attribute_group nvme_ns_id_attr_group = {
Christoph Hellwig5b85b822017-11-09 13:51:03 +01003431 .attrs = nvme_ns_id_attrs,
3432 .is_visible = nvme_ns_id_attrs_are_visible,
Keith Busch2b9b6e82015-12-22 10:10:45 -07003433};
3434
Hannes Reinecke33b14f672018-09-28 08:17:20 +02003435const struct attribute_group *nvme_ns_id_attr_groups[] = {
3436 &nvme_ns_id_attr_group,
3437#ifdef CONFIG_NVM
3438 &nvme_nvm_attr_group,
3439#endif
3440 NULL,
3441};
3442
Ming Lin931e1c22016-02-26 13:24:19 -08003443#define nvme_show_str_function(field) \
Keith Busch779ff7562016-01-12 15:09:31 -07003444static ssize_t field##_show(struct device *dev, \
3445 struct device_attribute *attr, char *buf) \
3446{ \
3447 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01003448 return sprintf(buf, "%.*s\n", \
3449 (int)sizeof(ctrl->subsys->field), ctrl->subsys->field); \
Keith Busch779ff7562016-01-12 15:09:31 -07003450} \
3451static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3452
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01003453nvme_show_str_function(model);
3454nvme_show_str_function(serial);
3455nvme_show_str_function(firmware_rev);
3456
Ming Lin931e1c22016-02-26 13:24:19 -08003457#define nvme_show_int_function(field) \
3458static ssize_t field##_show(struct device *dev, \
3459 struct device_attribute *attr, char *buf) \
3460{ \
3461 struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
3462 return sprintf(buf, "%d\n", ctrl->field); \
3463} \
3464static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3465
Ming Lin931e1c22016-02-26 13:24:19 -08003466nvme_show_int_function(cntlid);
Hannes Reinecke103e5152018-11-16 09:22:29 +01003467nvme_show_int_function(numa_node);
James Smart2b1ff252019-09-24 14:22:08 -07003468nvme_show_int_function(queue_count);
3469nvme_show_int_function(sqsize);
Keith Busch779ff7562016-01-12 15:09:31 -07003470
Ming Lin1a353d82016-06-13 16:45:24 +02003471static ssize_t nvme_sysfs_delete(struct device *dev,
3472 struct device_attribute *attr, const char *buf,
3473 size_t count)
3474{
3475 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3476
Israel Rukshince151812020-03-24 17:29:43 +02003477 /* Can't delete non-created controllers */
3478 if (!ctrl->created)
3479 return -EBUSY;
3480
Ming Lin1a353d82016-06-13 16:45:24 +02003481 if (device_remove_file_self(dev, attr))
Christoph Hellwigc5017e82017-10-29 10:44:29 +02003482 nvme_delete_ctrl_sync(ctrl);
Ming Lin1a353d82016-06-13 16:45:24 +02003483 return count;
3484}
3485static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3486
3487static ssize_t nvme_sysfs_show_transport(struct device *dev,
3488 struct device_attribute *attr,
3489 char *buf)
3490{
3491 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3492
3493 return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
3494}
3495static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3496
Sagi Grimberg8432bdb22016-11-28 01:47:40 +02003497static ssize_t nvme_sysfs_show_state(struct device *dev,
3498 struct device_attribute *attr,
3499 char *buf)
3500{
3501 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3502 static const char *const state_name[] = {
3503 [NVME_CTRL_NEW] = "new",
3504 [NVME_CTRL_LIVE] = "live",
3505 [NVME_CTRL_RESETTING] = "resetting",
Max Gurtovoyad6a0a52018-01-31 18:31:24 +02003506 [NVME_CTRL_CONNECTING] = "connecting",
Sagi Grimberg8432bdb22016-11-28 01:47:40 +02003507 [NVME_CTRL_DELETING] = "deleting",
Sagi Grimbergecca390e2020-07-22 16:32:19 -07003508 [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
Sagi Grimberg8432bdb22016-11-28 01:47:40 +02003509 [NVME_CTRL_DEAD] = "dead",
3510 };
3511
3512 if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3513 state_name[ctrl->state])
3514 return sprintf(buf, "%s\n", state_name[ctrl->state]);
3515
3516 return sprintf(buf, "unknown state\n");
3517}
3518
3519static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3520
Ming Lin1a353d82016-06-13 16:45:24 +02003521static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3522 struct device_attribute *attr,
3523 char *buf)
3524{
3525 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3526
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01003527 return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subsys->subnqn);
Ming Lin1a353d82016-06-13 16:45:24 +02003528}
3529static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3530
Sagi Grimberg76171c62020-02-07 17:13:53 -08003531static ssize_t nvme_sysfs_show_hostnqn(struct device *dev,
3532 struct device_attribute *attr,
3533 char *buf)
3534{
3535 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3536
3537 return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->opts->host->nqn);
3538}
3539static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL);
3540
Sagi Grimberg45fb19f2020-02-07 17:13:54 -08003541static ssize_t nvme_sysfs_show_hostid(struct device *dev,
3542 struct device_attribute *attr,
3543 char *buf)
3544{
3545 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3546
3547 return snprintf(buf, PAGE_SIZE, "%pU\n", &ctrl->opts->host->id);
3548}
3549static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL);
3550
Ming Lin1a353d82016-06-13 16:45:24 +02003551static ssize_t nvme_sysfs_show_address(struct device *dev,
3552 struct device_attribute *attr,
3553 char *buf)
3554{
3555 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3556
3557 return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3558}
3559static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3560
Sagi Grimberg764075f2020-07-05 00:57:55 -07003561static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev,
3562 struct device_attribute *attr, char *buf)
3563{
3564 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3565 struct nvmf_ctrl_options *opts = ctrl->opts;
3566
3567 if (ctrl->opts->max_reconnects == -1)
3568 return sprintf(buf, "off\n");
3569 return sprintf(buf, "%d\n",
3570 opts->max_reconnects * opts->reconnect_delay);
3571}
3572
3573static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev,
3574 struct device_attribute *attr, const char *buf, size_t count)
3575{
3576 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3577 struct nvmf_ctrl_options *opts = ctrl->opts;
3578 int ctrl_loss_tmo, err;
3579
3580 err = kstrtoint(buf, 10, &ctrl_loss_tmo);
3581 if (err)
3582 return -EINVAL;
3583
3584 else if (ctrl_loss_tmo < 0)
3585 opts->max_reconnects = -1;
3586 else
3587 opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
3588 opts->reconnect_delay);
3589 return count;
3590}
3591static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO | S_IWUSR,
3592 nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store);
3593
3594static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
3595 struct device_attribute *attr, char *buf)
3596{
3597 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3598
3599 if (ctrl->opts->reconnect_delay == -1)
3600 return sprintf(buf, "off\n");
3601 return sprintf(buf, "%d\n", ctrl->opts->reconnect_delay);
3602}
3603
3604static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
3605 struct device_attribute *attr, const char *buf, size_t count)
3606{
3607 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3608 unsigned int v;
3609 int err;
3610
3611 err = kstrtou32(buf, 10, &v);
Dan Carpentereca9e822020-07-14 13:57:32 +03003612 if (err)
3613 return err;
Sagi Grimberg764075f2020-07-05 00:57:55 -07003614
3615 ctrl->opts->reconnect_delay = v;
3616 return count;
3617}
3618static DEVICE_ATTR(reconnect_delay, S_IRUGO | S_IWUSR,
3619 nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store);
3620
Keith Busch779ff7562016-01-12 15:09:31 -07003621static struct attribute *nvme_dev_attrs[] = {
3622 &dev_attr_reset_controller.attr,
Keith Busch9ec3bb22016-04-29 15:45:18 -06003623 &dev_attr_rescan_controller.attr,
Keith Busch779ff7562016-01-12 15:09:31 -07003624 &dev_attr_model.attr,
3625 &dev_attr_serial.attr,
3626 &dev_attr_firmware_rev.attr,
Ming Lin931e1c22016-02-26 13:24:19 -08003627 &dev_attr_cntlid.attr,
Ming Lin1a353d82016-06-13 16:45:24 +02003628 &dev_attr_delete_controller.attr,
3629 &dev_attr_transport.attr,
3630 &dev_attr_subsysnqn.attr,
3631 &dev_attr_address.attr,
Sagi Grimberg8432bdb22016-11-28 01:47:40 +02003632 &dev_attr_state.attr,
Hannes Reinecke103e5152018-11-16 09:22:29 +01003633 &dev_attr_numa_node.attr,
James Smart2b1ff252019-09-24 14:22:08 -07003634 &dev_attr_queue_count.attr,
3635 &dev_attr_sqsize.attr,
Sagi Grimberg76171c62020-02-07 17:13:53 -08003636 &dev_attr_hostnqn.attr,
Sagi Grimberg45fb19f2020-02-07 17:13:54 -08003637 &dev_attr_hostid.attr,
Sagi Grimberg764075f2020-07-05 00:57:55 -07003638 &dev_attr_ctrl_loss_tmo.attr,
3639 &dev_attr_reconnect_delay.attr,
Keith Busch779ff7562016-01-12 15:09:31 -07003640 NULL
3641};
3642
Ming Lin1a353d82016-06-13 16:45:24 +02003643static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3644 struct attribute *a, int n)
3645{
3646 struct device *dev = container_of(kobj, struct device, kobj);
3647 struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3648
Christoph Hellwig49d3d502017-06-26 12:39:03 +02003649 if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3650 return 0;
3651 if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3652 return 0;
Sagi Grimberg76171c62020-02-07 17:13:53 -08003653 if (a == &dev_attr_hostnqn.attr && !ctrl->opts)
3654 return 0;
Sagi Grimberg45fb19f2020-02-07 17:13:54 -08003655 if (a == &dev_attr_hostid.attr && !ctrl->opts)
3656 return 0;
Ming Lin1a353d82016-06-13 16:45:24 +02003657
3658 return a->mode;
3659}
3660
Keith Busch779ff7562016-01-12 15:09:31 -07003661static struct attribute_group nvme_dev_attrs_group = {
Ming Lin1a353d82016-06-13 16:45:24 +02003662 .attrs = nvme_dev_attrs,
3663 .is_visible = nvme_dev_attrs_are_visible,
Keith Busch779ff7562016-01-12 15:09:31 -07003664};
3665
3666static const struct attribute_group *nvme_dev_attr_groups[] = {
3667 &nvme_dev_attrs_group,
3668 NULL,
3669};
3670
Christoph Hellwig026d2ef2020-03-25 14:19:36 +01003671static struct nvme_ns_head *nvme_find_ns_head(struct nvme_subsystem *subsys,
Christoph Hellwiged754e52017-11-09 13:50:43 +01003672 unsigned nsid)
3673{
3674 struct nvme_ns_head *h;
3675
3676 lockdep_assert_held(&subsys->lock);
3677
3678 list_for_each_entry(h, &subsys->nsheads, entry) {
3679 if (h->ns_id == nsid && kref_get_unless_zero(&h->ref))
3680 return h;
3681 }
3682
3683 return NULL;
3684}
3685
3686static int __nvme_check_ids(struct nvme_subsystem *subsys,
3687 struct nvme_ns_head *new)
3688{
3689 struct nvme_ns_head *h;
3690
3691 lockdep_assert_held(&subsys->lock);
3692
3693 list_for_each_entry(h, &subsys->nsheads, entry) {
3694 if (nvme_ns_ids_valid(&new->ids) &&
3695 nvme_ns_ids_equal(&new->ids, &h->ids))
3696 return -EINVAL;
3697 }
3698
3699 return 0;
3700}
3701
3702static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
Keith Busch03f8ceb2020-04-03 09:24:09 -07003703 unsigned nsid, struct nvme_ns_ids *ids)
Christoph Hellwiged754e52017-11-09 13:50:43 +01003704{
3705 struct nvme_ns_head *head;
Christoph Hellwigf3334442018-09-11 09:51:29 +02003706 size_t size = sizeof(*head);
Christoph Hellwiged754e52017-11-09 13:50:43 +01003707 int ret = -ENOMEM;
3708
Christoph Hellwigf3334442018-09-11 09:51:29 +02003709#ifdef CONFIG_NVME_MULTIPATH
3710 size += num_possible_nodes() * sizeof(struct nvme_ns *);
3711#endif
3712
3713 head = kzalloc(size, GFP_KERNEL);
Christoph Hellwiged754e52017-11-09 13:50:43 +01003714 if (!head)
3715 goto out;
3716 ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL);
3717 if (ret < 0)
3718 goto out_free_head;
3719 head->instance = ret;
3720 INIT_LIST_HEAD(&head->list);
Max Gurtovoyfd92c772018-04-12 09:16:12 -06003721 ret = init_srcu_struct(&head->srcu);
3722 if (ret)
3723 goto out_ida_remove;
Christoph Hellwiged754e52017-11-09 13:50:43 +01003724 head->subsys = ctrl->subsys;
3725 head->ns_id = nsid;
Christoph Hellwig43fcd9e2020-03-25 14:19:37 +01003726 head->ids = *ids;
Christoph Hellwiged754e52017-11-09 13:50:43 +01003727 kref_init(&head->ref);
3728
Christoph Hellwiged754e52017-11-09 13:50:43 +01003729 ret = __nvme_check_ids(ctrl->subsys, head);
3730 if (ret) {
3731 dev_err(ctrl->device,
3732 "duplicate IDs for nsid %d\n", nsid);
3733 goto out_cleanup_srcu;
3734 }
3735
Keith Buschbe93e872020-06-29 12:06:40 -07003736 if (head->ids.csi) {
3737 ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
3738 if (ret)
3739 goto out_cleanup_srcu;
3740 } else
3741 head->effects = ctrl->effects;
3742
Christoph Hellwig32acab32017-11-02 12:59:30 +01003743 ret = nvme_mpath_alloc_disk(ctrl, head);
3744 if (ret)
3745 goto out_cleanup_srcu;
3746
Christoph Hellwiged754e52017-11-09 13:50:43 +01003747 list_add_tail(&head->entry, &ctrl->subsys->nsheads);
Jianchao Wang12d9f072018-05-04 16:01:57 +08003748
3749 kref_get(&ctrl->subsys->ref);
3750
Christoph Hellwiged754e52017-11-09 13:50:43 +01003751 return head;
3752out_cleanup_srcu:
3753 cleanup_srcu_struct(&head->srcu);
Max Gurtovoyfd92c772018-04-12 09:16:12 -06003754out_ida_remove:
Christoph Hellwiged754e52017-11-09 13:50:43 +01003755 ida_simple_remove(&ctrl->subsys->ns_ida, head->instance);
3756out_free_head:
3757 kfree(head);
3758out:
Sagi Grimberg538af882019-08-02 18:16:12 -07003759 if (ret > 0)
3760 ret = blk_status_to_errno(nvme_error_status(ret));
Christoph Hellwiged754e52017-11-09 13:50:43 +01003761 return ERR_PTR(ret);
3762}
3763
3764static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
Baegjae Sung9bd82b12018-02-28 16:06:04 +09003765 struct nvme_id_ns *id)
Christoph Hellwiged754e52017-11-09 13:50:43 +01003766{
3767 struct nvme_ctrl *ctrl = ns->ctrl;
Keith Busch92decf12020-04-03 10:53:46 -07003768 bool is_shared = id->nmic & NVME_NS_NMIC_SHARED;
Christoph Hellwiged754e52017-11-09 13:50:43 +01003769 struct nvme_ns_head *head = NULL;
Christoph Hellwig43fcd9e2020-03-25 14:19:37 +01003770 struct nvme_ns_ids ids;
Christoph Hellwiged754e52017-11-09 13:50:43 +01003771 int ret = 0;
3772
Christoph Hellwig43fcd9e2020-03-25 14:19:37 +01003773 ret = nvme_report_ns_ids(ctrl, nsid, id, &ids);
Christoph Hellwig6623c5b2020-04-22 09:59:08 +02003774 if (ret) {
3775 if (ret < 0)
3776 return ret;
3777 return blk_status_to_errno(nvme_error_status(ret));
3778 }
Christoph Hellwig43fcd9e2020-03-25 14:19:37 +01003779
Christoph Hellwiged754e52017-11-09 13:50:43 +01003780 mutex_lock(&ctrl->subsys->lock);
Keith Busch9ad19272020-04-09 09:09:01 -07003781 head = nvme_find_ns_head(ctrl->subsys, nsid);
Christoph Hellwiged754e52017-11-09 13:50:43 +01003782 if (!head) {
Keith Busch03f8ceb2020-04-03 09:24:09 -07003783 head = nvme_alloc_ns_head(ctrl, nsid, &ids);
Christoph Hellwiged754e52017-11-09 13:50:43 +01003784 if (IS_ERR(head)) {
3785 ret = PTR_ERR(head);
3786 goto out_unlock;
3787 }
Keith Busch0c284db2020-04-09 09:09:02 -07003788 head->shared = is_shared;
Christoph Hellwiged754e52017-11-09 13:50:43 +01003789 } else {
Christoph Hellwig6623c5b2020-04-22 09:59:08 +02003790 ret = -EINVAL;
Keith Busch0c284db2020-04-09 09:09:02 -07003791 if (!is_shared || !head->shared) {
Keith Busch9ad19272020-04-09 09:09:01 -07003792 dev_err(ctrl->device,
Christoph Hellwig6623c5b2020-04-22 09:59:08 +02003793 "Duplicate unshared namespace %d\n", nsid);
3794 goto out_put_ns_head;
Keith Busch9ad19272020-04-09 09:09:01 -07003795 }
Christoph Hellwiged754e52017-11-09 13:50:43 +01003796 if (!nvme_ns_ids_equal(&head->ids, &ids)) {
3797 dev_err(ctrl->device,
3798 "IDs don't match for shared namespace %d\n",
3799 nsid);
Christoph Hellwig6623c5b2020-04-22 09:59:08 +02003800 goto out_put_ns_head;
Christoph Hellwiged754e52017-11-09 13:50:43 +01003801 }
Christoph Hellwiged754e52017-11-09 13:50:43 +01003802 }
3803
3804 list_add_tail(&ns->siblings, &head->list);
3805 ns->head = head;
Christoph Hellwig6623c5b2020-04-22 09:59:08 +02003806 mutex_unlock(&ctrl->subsys->lock);
3807 return 0;
Christoph Hellwiged754e52017-11-09 13:50:43 +01003808
Christoph Hellwig6623c5b2020-04-22 09:59:08 +02003809out_put_ns_head:
3810 nvme_put_ns_head(head);
Christoph Hellwiged754e52017-11-09 13:50:43 +01003811out_unlock:
3812 mutex_unlock(&ctrl->subsys->lock);
3813 return ret;
3814}
3815
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003816static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
3817{
3818 struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
3819 struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
3820
Christoph Hellwiged754e52017-11-09 13:50:43 +01003821 return nsa->head->ns_id - nsb->head->ns_id;
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003822}
3823
Logan Gunthorpe24493b82020-07-24 11:25:16 -06003824struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003825{
Keith Busch32f0c4a2016-07-13 11:45:02 -06003826 struct nvme_ns *ns, *ret = NULL;
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003827
Jianchao Wang765cc0312018-02-12 20:54:46 +08003828 down_read(&ctrl->namespaces_rwsem);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003829 list_for_each_entry(ns, &ctrl->namespaces, list) {
Christoph Hellwiged754e52017-11-09 13:50:43 +01003830 if (ns->head->ns_id == nsid) {
Christoph Hellwig2dd41222017-10-18 13:20:01 +02003831 if (!kref_get_unless_zero(&ns->kref))
3832 continue;
Keith Busch32f0c4a2016-07-13 11:45:02 -06003833 ret = ns;
3834 break;
3835 }
Christoph Hellwiged754e52017-11-09 13:50:43 +01003836 if (ns->head->ns_id > nsid)
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003837 break;
3838 }
Jianchao Wang765cc0312018-02-12 20:54:46 +08003839 up_read(&ctrl->namespaces_rwsem);
Keith Busch32f0c4a2016-07-13 11:45:02 -06003840 return ret;
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003841}
Logan Gunthorpe24493b82020-07-24 11:25:16 -06003842EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003843
Edmund Nadolskiadce7e92019-11-27 10:17:43 -07003844static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003845{
3846 struct nvme_ns *ns;
3847 struct gendisk *disk;
Matias Bjørlingac81bfa92016-09-16 14:25:04 +02003848 struct nvme_id_ns *id;
3849 char disk_name[DISK_NAME_LEN];
Hannes Reineckeab4ab092019-02-19 13:13:57 +01003850 int node = ctrl->numa_node, flags = GENHD_FL_EXT_DEVT, ret;
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003851
3852 ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
3853 if (!ns)
Edmund Nadolskiadce7e92019-11-27 10:17:43 -07003854 return;
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003855
3856 ns->queue = blk_mq_init_queue(ctrl->tagset);
Edmund Nadolskiadce7e92019-11-27 10:17:43 -07003857 if (IS_ERR(ns->queue))
Christoph Hellwiged754e52017-11-09 13:50:43 +01003858 goto out_free_ns;
Logan Gunthorpee0596ab2018-10-04 15:27:44 -06003859
Minwoo Im7d30c812019-07-12 02:04:47 +09003860 if (ctrl->opts && ctrl->opts->data_digest)
Mikhail Skorzhinskii958f2a02019-07-04 09:59:18 +02003861 ns->queue->backing_dev_info->capabilities
3862 |= BDI_CAP_STABLE_WRITES;
3863
Bart Van Assche8b904b52018-03-07 17:10:10 -08003864 blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
Logan Gunthorpee0596ab2018-10-04 15:27:44 -06003865 if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)
3866 blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
3867
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003868 ns->queue->queuedata = ns;
3869 ns->ctrl = ctrl;
3870
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003871 kref_init(&ns->kref);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003872 ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003873
3874 blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
Christoph Hellwigda358252016-03-02 18:07:11 +01003875 nvme_set_queue_limits(ctrl, ns->queue);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003876
Sagi Grimberg331813f2019-08-02 18:11:42 -07003877 ret = nvme_identify_ns(ctrl, nsid, &id);
3878 if (ret)
Matias Bjørlingac81bfa92016-09-16 14:25:04 +02003879 goto out_free_queue;
3880
Edmund Nadolskiadce7e92019-11-27 10:17:43 -07003881 if (id->ncap == 0) /* no namespace (legacy quirk) */
Christoph Hellwigcdbff4f2017-08-16 16:14:47 +02003882 goto out_free_id;
3883
Hannes Reineckeab4ab092019-02-19 13:13:57 +01003884 ret = nvme_init_ns_head(ns, nsid, id);
3885 if (ret)
Christoph Hellwiged754e52017-11-09 13:50:43 +01003886 goto out_free_id;
Keith Buscha785dbc2018-04-26 14:22:41 -06003887 nvme_set_disk_name(disk_name, ns, ctrl, &flags);
Christoph Hellwigcdbff4f2017-08-16 16:14:47 +02003888
Matias Bjørling3dc87dd2016-11-28 22:38:53 +01003889 disk = alloc_disk_node(0, node);
Edmund Nadolskiadce7e92019-11-27 10:17:43 -07003890 if (!disk)
Christoph Hellwiged754e52017-11-09 13:50:43 +01003891 goto out_unlink_ns;
Matias Bjørling3dc87dd2016-11-28 22:38:53 +01003892
3893 disk->fops = &nvme_fops;
3894 disk->private_data = ns;
3895 disk->queue = ns->queue;
Christoph Hellwig32acab32017-11-02 12:59:30 +01003896 disk->flags = flags;
Matias Bjørling3dc87dd2016-11-28 22:38:53 +01003897 memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
3898 ns->disk = disk;
3899
Max Gurtovoy33cfdc22020-05-19 17:05:53 +03003900 if (__nvme_revalidate_disk(disk, id))
Niklas Cassel108a5852020-06-07 13:45:20 +02003901 goto out_put_disk;
Matias Bjørling3dc87dd2016-11-28 22:38:53 +01003902
Matias Bjørling85136c02018-12-11 20:16:20 +01003903 if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
Hannes Reineckeab4ab092019-02-19 13:13:57 +01003904 ret = nvme_nvm_register(ns, disk_name, node);
3905 if (ret) {
Matias Bjørling85136c02018-12-11 20:16:20 +01003906 dev_warn(ctrl->device, "LightNVM init failure\n");
3907 goto out_put_disk;
3908 }
3909 }
3910
Jianchao Wang765cc0312018-02-12 20:54:46 +08003911 down_write(&ctrl->namespaces_rwsem);
Keith Busch32f0c4a2016-07-13 11:45:02 -06003912 list_add_tail(&ns->list, &ctrl->namespaces);
Jianchao Wang765cc0312018-02-12 20:54:46 +08003913 up_write(&ctrl->namespaces_rwsem);
Keith Busch32f0c4a2016-07-13 11:45:02 -06003914
Christoph Hellwigd22524a2017-10-18 13:25:42 +02003915 nvme_get_ctrl(ctrl);
Matias Bjørlingac81bfa92016-09-16 14:25:04 +02003916
Hannes Reinecke33b14f672018-09-28 08:17:20 +02003917 device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
Christoph Hellwig32acab32017-11-02 12:59:30 +01003918
Christoph Hellwig0d0b6602018-05-14 08:48:54 +02003919 nvme_mpath_add_disk(ns, id);
Akinobu Mitaa3646452019-06-20 08:49:02 +02003920 nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
Christoph Hellwig0d0b6602018-05-14 08:48:54 +02003921 kfree(id);
3922
Edmund Nadolskiadce7e92019-11-27 10:17:43 -07003923 return;
Matias Bjørling85136c02018-12-11 20:16:20 +01003924 out_put_disk:
Niklas Cassel132be622020-04-27 14:34:41 +02003925 /* prevent double queue cleanup */
3926 ns->disk->queue = NULL;
Matias Bjørling85136c02018-12-11 20:16:20 +01003927 put_disk(ns->disk);
Christoph Hellwiged754e52017-11-09 13:50:43 +01003928 out_unlink_ns:
3929 mutex_lock(&ctrl->subsys->lock);
3930 list_del_rcu(&ns->siblings);
Keith Buschd5675722020-04-09 09:08:59 -07003931 if (list_empty(&ns->head->list))
3932 list_del_init(&ns->head->entry);
Christoph Hellwiged754e52017-11-09 13:50:43 +01003933 mutex_unlock(&ctrl->subsys->lock);
Sagi Grimberga63b8372019-03-13 18:54:57 +01003934 nvme_put_ns_head(ns->head);
Matias Bjørlingac81bfa92016-09-16 14:25:04 +02003935 out_free_id:
3936 kfree(id);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003937 out_free_queue:
3938 blk_cleanup_queue(ns->queue);
3939 out_free_ns:
3940 kfree(ns);
3941}
3942
3943static void nvme_ns_remove(struct nvme_ns *ns)
3944{
Keith Busch646017a2016-02-24 09:15:54 -07003945 if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
3946 return;
3947
Akinobu Mitaa3646452019-06-20 08:49:02 +02003948 nvme_fault_inject_fini(&ns->fault_inject);
Anton Eidelman2181e452019-06-20 08:48:10 +02003949
3950 mutex_lock(&ns->ctrl->subsys->lock);
3951 list_del_rcu(&ns->siblings);
Keith Buschd5675722020-04-09 09:08:59 -07003952 if (list_empty(&ns->head->list))
3953 list_del_init(&ns->head->entry);
Anton Eidelman2181e452019-06-20 08:48:10 +02003954 mutex_unlock(&ns->ctrl->subsys->lock);
Keith Buschd5675722020-04-09 09:08:59 -07003955
Anton Eidelman2181e452019-06-20 08:48:10 +02003956 synchronize_rcu(); /* guarantee not available in head->list */
3957 nvme_mpath_clear_current_path(ns);
3958 synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */
3959
Christoph Hellwig3913f4f2020-07-08 16:18:27 +02003960 if (ns->disk->flags & GENHD_FL_UP) {
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003961 del_gendisk(ns->disk);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003962 blk_cleanup_queue(ns->queue);
Ming Leibd9f5d652017-12-06 18:30:09 +08003963 if (blk_get_integrity(ns->disk))
3964 blk_integrity_unregister(ns->disk);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003965 }
Keith Busch32f0c4a2016-07-13 11:45:02 -06003966
Jianchao Wang765cc0312018-02-12 20:54:46 +08003967 down_write(&ns->ctrl->namespaces_rwsem);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003968 list_del_init(&ns->list);
Jianchao Wang765cc0312018-02-12 20:54:46 +08003969 up_write(&ns->ctrl->namespaces_rwsem);
Keith Busch32f0c4a2016-07-13 11:45:02 -06003970
Sagi Grimberg479a3222017-12-21 15:07:27 +02003971 nvme_mpath_check_last_path(ns);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01003972 nvme_put_ns(ns);
3973}
3974
Christoph Hellwig4450ba32020-04-04 10:30:32 +02003975static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
3976{
3977 struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
3978
3979 if (ns) {
3980 nvme_ns_remove(ns);
3981 nvme_put_ns(ns);
3982 }
3983}
3984
Keith Busch540c8012015-10-22 15:45:06 -06003985static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3986{
3987 struct nvme_ns *ns;
3988
Keith Busch32f0c4a2016-07-13 11:45:02 -06003989 ns = nvme_find_get_ns(ctrl, nsid);
Keith Busch540c8012015-10-22 15:45:06 -06003990 if (ns) {
Christoph Hellwig3913f4f2020-07-08 16:18:27 +02003991 if (revalidate_disk(ns->disk))
Keith Busch540c8012015-10-22 15:45:06 -06003992 nvme_ns_remove(ns);
Keith Busch32f0c4a2016-07-13 11:45:02 -06003993 nvme_put_ns(ns);
Keith Busch540c8012015-10-22 15:45:06 -06003994 } else
3995 nvme_alloc_ns(ctrl, nsid);
3996}
3997
Sunad Bhandary47b0e502016-05-27 15:59:43 +05303998static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
3999 unsigned nsid)
4000{
4001 struct nvme_ns *ns, *next;
Jianchao Wang6f8e0d72018-02-12 20:54:44 +08004002 LIST_HEAD(rm_list);
Sunad Bhandary47b0e502016-05-27 15:59:43 +05304003
Jianchao Wang765cc0312018-02-12 20:54:46 +08004004 down_write(&ctrl->namespaces_rwsem);
Sunad Bhandary47b0e502016-05-27 15:59:43 +05304005 list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
Scott Bauercf39a6b2018-06-29 13:03:28 -06004006 if (ns->head->ns_id > nsid || test_bit(NVME_NS_DEAD, &ns->flags))
Jianchao Wang6f8e0d72018-02-12 20:54:44 +08004007 list_move_tail(&ns->list, &rm_list);
Sunad Bhandary47b0e502016-05-27 15:59:43 +05304008 }
Jianchao Wang765cc0312018-02-12 20:54:46 +08004009 up_write(&ctrl->namespaces_rwsem);
Jianchao Wang6f8e0d72018-02-12 20:54:44 +08004010
4011 list_for_each_entry_safe(ns, next, &rm_list, list)
4012 nvme_ns_remove(ns);
4013
Sunad Bhandary47b0e502016-05-27 15:59:43 +05304014}
4015
Christoph Hellwig4005f282020-04-04 10:31:35 +02004016static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
Keith Busch540c8012015-10-22 15:45:06 -06004017{
Christoph Hellwigaec459b2020-04-04 10:34:21 +02004018 const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
Keith Busch540c8012015-10-22 15:45:06 -06004019 __le32 *ns_list;
Christoph Hellwig4005f282020-04-04 10:31:35 +02004020 u32 prev = 0;
4021 int ret = 0, i;
Keith Busch540c8012015-10-22 15:45:06 -06004022
Christoph Hellwig25dcaa92020-04-04 10:16:03 +02004023 if (nvme_ctrl_limited_cns(ctrl))
4024 return -EOPNOTSUPP;
Keith Busch540c8012015-10-22 15:45:06 -06004025
Minwoo Im42595eb2018-02-08 22:56:31 +09004026 ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
Keith Busch540c8012015-10-22 15:45:06 -06004027 if (!ns_list)
4028 return -ENOMEM;
4029
Christoph Hellwig4005f282020-04-04 10:31:35 +02004030 for (;;) {
Keith Busch540c8012015-10-22 15:45:06 -06004031 ret = nvme_identify_ns_list(ctrl, prev, ns_list);
4032 if (ret)
Sunad Bhandary47b0e502016-05-27 15:59:43 +05304033 goto free;
Keith Busch540c8012015-10-22 15:45:06 -06004034
Christoph Hellwigaec459b2020-04-04 10:34:21 +02004035 for (i = 0; i < nr_entries; i++) {
Christoph Hellwig4005f282020-04-04 10:31:35 +02004036 u32 nsid = le32_to_cpu(ns_list[i]);
4037
4038 if (!nsid) /* end of the list? */
Keith Busch540c8012015-10-22 15:45:06 -06004039 goto out;
Keith Busch540c8012015-10-22 15:45:06 -06004040 nvme_validate_ns(ctrl, nsid);
Christoph Hellwig4450ba32020-04-04 10:30:32 +02004041 while (++prev < nsid)
4042 nvme_ns_remove_by_nsid(ctrl, prev);
Keith Busch540c8012015-10-22 15:45:06 -06004043 }
Keith Busch540c8012015-10-22 15:45:06 -06004044 }
4045 out:
Sunad Bhandary47b0e502016-05-27 15:59:43 +05304046 nvme_remove_invalid_namespaces(ctrl, prev);
4047 free:
Keith Busch540c8012015-10-22 15:45:06 -06004048 kfree(ns_list);
4049 return ret;
4050}
4051
Christoph Hellwig4005f282020-04-04 10:31:35 +02004052static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004053{
Christoph Hellwig4005f282020-04-04 10:31:35 +02004054 struct nvme_id_ctrl *id;
4055 u32 nn, i;
4056
4057 if (nvme_identify_ctrl(ctrl, &id))
4058 return;
4059 nn = le32_to_cpu(id->nn);
4060 kfree(id);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004061
Keith Busch540c8012015-10-22 15:45:06 -06004062 for (i = 1; i <= nn; i++)
4063 nvme_validate_ns(ctrl, i);
4064
Sunad Bhandary47b0e502016-05-27 15:59:43 +05304065 nvme_remove_invalid_namespaces(ctrl, nn);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004066}
4067
Christoph Hellwigf493af32018-06-07 13:47:33 +02004068static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
Christoph Hellwig30d90962018-05-25 18:17:41 +02004069{
4070 size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4071 __le32 *log;
Christoph Hellwigf493af32018-06-07 13:47:33 +02004072 int error;
Christoph Hellwig30d90962018-05-25 18:17:41 +02004073
4074 log = kzalloc(log_size, GFP_KERNEL);
4075 if (!log)
Christoph Hellwigf493af32018-06-07 13:47:33 +02004076 return;
Christoph Hellwig30d90962018-05-25 18:17:41 +02004077
Christoph Hellwigf493af32018-06-07 13:47:33 +02004078 /*
4079 * We need to read the log to clear the AEN, but we don't want to rely
4080 * on it for the changed namespace information as userspace could have
4081 * raced with us in reading the log page, which could cause us to miss
4082 * updates.
4083 */
Keith Buschbe93e872020-06-29 12:06:40 -07004084 error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4085 NVME_CSI_NVM, log, log_size, 0);
Christoph Hellwigf493af32018-06-07 13:47:33 +02004086 if (error)
Christoph Hellwig30d90962018-05-25 18:17:41 +02004087 dev_warn(ctrl->device,
4088 "reading changed ns log failed: %d\n", error);
Christoph Hellwig30d90962018-05-25 18:17:41 +02004089
Christoph Hellwig30d90962018-05-25 18:17:41 +02004090 kfree(log);
Christoph Hellwig30d90962018-05-25 18:17:41 +02004091}
4092
Christoph Hellwig5955be22016-04-26 13:51:59 +02004093static void nvme_scan_work(struct work_struct *work)
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004094{
Christoph Hellwig5955be22016-04-26 13:51:59 +02004095 struct nvme_ctrl *ctrl =
4096 container_of(work, struct nvme_ctrl, scan_work);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004097
Keith Busch5d02a5c2019-09-03 09:22:24 -06004098 /* No tagset on a live ctrl means IO queues could not created */
4099 if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
Christoph Hellwig5955be22016-04-26 13:51:59 +02004100 return;
4101
Dan Carpenter77016192018-06-07 11:27:41 +03004102 if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
Christoph Hellwig30d90962018-05-25 18:17:41 +02004103 dev_info(ctrl->device, "rescanning namespaces.\n");
Christoph Hellwigf493af32018-06-07 13:47:33 +02004104 nvme_clear_changed_ns_log(ctrl);
Christoph Hellwig30d90962018-05-25 18:17:41 +02004105 }
4106
Keith Busche7ad43c2019-01-28 09:46:07 -07004107 mutex_lock(&ctrl->scan_lock);
Christoph Hellwig4005f282020-04-04 10:31:35 +02004108 if (nvme_scan_ns_list(ctrl) != 0)
4109 nvme_scan_ns_sequential(ctrl);
Keith Busche7ad43c2019-01-28 09:46:07 -07004110 mutex_unlock(&ctrl->scan_lock);
Christoph Hellwig25dcaa92020-04-04 10:16:03 +02004111
Jianchao Wang765cc0312018-02-12 20:54:46 +08004112 down_write(&ctrl->namespaces_rwsem);
Keith Busch540c8012015-10-22 15:45:06 -06004113 list_sort(NULL, &ctrl->namespaces, ns_cmp);
Jianchao Wang765cc0312018-02-12 20:54:46 +08004114 up_write(&ctrl->namespaces_rwsem);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004115}
Christoph Hellwig5955be22016-04-26 13:51:59 +02004116
Keith Busch32f0c4a2016-07-13 11:45:02 -06004117/*
4118 * This function iterates the namespace list unlocked to allow recovery from
4119 * controller failure. It is up to the caller to ensure the namespace list is
4120 * not modified by scan work while this function is executing.
4121 */
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004122void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4123{
4124 struct nvme_ns *ns, *next;
Jianchao Wang6f8e0d72018-02-12 20:54:44 +08004125 LIST_HEAD(ns_list);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004126
Sagi Grimberg0157ec82019-07-25 11:56:57 -07004127 /*
4128 * make sure to requeue I/O to all namespaces as these
4129 * might result from the scan itself and must complete
4130 * for the scan_work to make progress
4131 */
4132 nvme_mpath_clear_ctrl_paths(ctrl);
4133
Sagi Grimbergf6c8e432018-11-21 15:17:37 -08004134 /* prevent racing with ns scanning */
4135 flush_work(&ctrl->scan_work);
4136
Keith Busch0ff9d4e2016-05-12 08:37:14 -06004137 /*
4138 * The dead states indicates the controller was not gracefully
4139 * disconnected. In that case, we won't be able to flush any data while
4140 * removing the namespaces' disks; fail all the queues now to avoid
4141 * potentially having to clean up the failed sync later.
4142 */
4143 if (ctrl->state == NVME_CTRL_DEAD)
4144 nvme_kill_queues(ctrl);
4145
Sagi Grimbergecca390e2020-07-22 16:32:19 -07004146 /* this is a no-op when called from the controller reset handler */
4147 nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4148
Jianchao Wang765cc0312018-02-12 20:54:46 +08004149 down_write(&ctrl->namespaces_rwsem);
Jianchao Wang6f8e0d72018-02-12 20:54:44 +08004150 list_splice_init(&ctrl->namespaces, &ns_list);
Jianchao Wang765cc0312018-02-12 20:54:46 +08004151 up_write(&ctrl->namespaces_rwsem);
Jianchao Wang6f8e0d72018-02-12 20:54:44 +08004152
4153 list_for_each_entry_safe(ns, next, &ns_list, list)
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004154 nvme_ns_remove(ns);
4155}
Ming Lin576d55d2016-02-10 10:03:32 -08004156EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004157
Sagi Grimberga42f42e2019-09-04 14:29:48 -07004158static int nvme_class_uevent(struct device *dev, struct kobj_uevent_env *env)
4159{
4160 struct nvme_ctrl *ctrl =
4161 container_of(dev, struct nvme_ctrl, ctrl_device);
4162 struct nvmf_ctrl_options *opts = ctrl->opts;
4163 int ret;
4164
4165 ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4166 if (ret)
4167 return ret;
4168
4169 if (opts) {
4170 ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4171 if (ret)
4172 return ret;
4173
4174 ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4175 opts->trsvcid ?: "none");
4176 if (ret)
4177 return ret;
4178
4179 ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4180 opts->host_traddr ?: "none");
4181 }
4182 return ret;
4183}
4184
Keith Busche3d78742017-11-07 15:13:14 -07004185static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4186{
4187 char *envp[2] = { NULL, NULL };
4188 u32 aen_result = ctrl->aen_result;
4189
4190 ctrl->aen_result = 0;
4191 if (!aen_result)
4192 return;
4193
4194 envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4195 if (!envp[0])
4196 return;
4197 kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4198 kfree(envp[0]);
4199}
4200
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004201static void nvme_async_event_work(struct work_struct *work)
4202{
4203 struct nvme_ctrl *ctrl =
4204 container_of(work, struct nvme_ctrl, async_event_work);
4205
Keith Busche3d78742017-11-07 15:13:14 -07004206 nvme_aen_uevent(ctrl);
Keith Buschad22c352017-11-07 15:13:12 -07004207 ctrl->ops->submit_async_event(ctrl);
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004208}
4209
Arnav Dawnb6dccf72017-07-12 16:10:40 +05304210static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4211{
4212
4213 u32 csts;
4214
4215 if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4216 return false;
4217
4218 if (csts == ~0)
4219 return false;
4220
4221 return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4222}
4223
4224static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4225{
Arnav Dawnb6dccf72017-07-12 16:10:40 +05304226 struct nvme_fw_slot_info_log *log;
4227
4228 log = kmalloc(sizeof(*log), GFP_KERNEL);
4229 if (!log)
4230 return;
4231
Keith Buschbe93e872020-06-29 12:06:40 -07004232 if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4233 log, sizeof(*log), 0))
Christoph Hellwig0e987192018-06-06 14:39:00 +02004234 dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
Arnav Dawnb6dccf72017-07-12 16:10:40 +05304235 kfree(log);
4236}
4237
4238static void nvme_fw_act_work(struct work_struct *work)
4239{
4240 struct nvme_ctrl *ctrl = container_of(work,
4241 struct nvme_ctrl, fw_act_work);
4242 unsigned long fw_act_timeout;
4243
4244 if (ctrl->mtfa)
4245 fw_act_timeout = jiffies +
4246 msecs_to_jiffies(ctrl->mtfa * 100);
4247 else
4248 fw_act_timeout = jiffies +
4249 msecs_to_jiffies(admin_timeout * 1000);
4250
4251 nvme_stop_queues(ctrl);
4252 while (nvme_ctrl_pp_status(ctrl)) {
4253 if (time_after(jiffies, fw_act_timeout)) {
4254 dev_warn(ctrl->device,
4255 "Fw activation timeout, reset controller\n");
Keith Busch4c75f872019-09-06 11:23:08 -06004256 nvme_try_sched_reset(ctrl);
4257 return;
Arnav Dawnb6dccf72017-07-12 16:10:40 +05304258 }
4259 msleep(100);
4260 }
4261
Keith Busch4c75f872019-09-06 11:23:08 -06004262 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
Arnav Dawnb6dccf72017-07-12 16:10:40 +05304263 return;
4264
4265 nvme_start_queues(ctrl);
Minwoo Ima806c6c2017-11-02 18:07:44 +09004266 /* read FW slot information to clear the AER */
Arnav Dawnb6dccf72017-07-12 16:10:40 +05304267 nvme_get_fw_slot_info(ctrl);
4268}
4269
Christoph Hellwig868c2392018-05-22 11:09:54 +02004270static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004271{
Chaitanya Kulkarni09bd1ff2018-09-17 10:47:06 -07004272 u32 aer_notice_type = (result & 0xff00) >> 8;
4273
Chaitanya Kulkarni521cfb82019-05-13 10:46:05 -07004274 trace_nvme_async_event(ctrl, aer_notice_type);
4275
Chaitanya Kulkarni09bd1ff2018-09-17 10:47:06 -07004276 switch (aer_notice_type) {
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004277 case NVME_AER_NOTICE_NS_CHANGED:
Dan Carpenter77016192018-06-07 11:27:41 +03004278 set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004279 nvme_queue_scan(ctrl);
4280 break;
Arnav Dawnb6dccf72017-07-12 16:10:40 +05304281 case NVME_AER_NOTICE_FW_ACT_STARTING:
Keith Busch4c75f872019-09-06 11:23:08 -06004282 /*
4283 * We are (ab)using the RESETTING state to prevent subsequent
4284 * recovery actions from interfering with the controller's
4285 * firmware activation.
4286 */
4287 if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
4288 queue_work(nvme_wq, &ctrl->fw_act_work);
Arnav Dawnb6dccf72017-07-12 16:10:40 +05304289 break;
Christoph Hellwig0d0b6602018-05-14 08:48:54 +02004290#ifdef CONFIG_NVME_MULTIPATH
4291 case NVME_AER_NOTICE_ANA:
4292 if (!ctrl->ana_log_buf)
4293 break;
4294 queue_work(nvme_wq, &ctrl->ana_work);
4295 break;
4296#endif
Sagi Grimberg85f8a432019-07-12 11:02:10 -07004297 case NVME_AER_NOTICE_DISC_CHANGED:
4298 ctrl->aen_result = result;
4299 break;
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004300 default:
4301 dev_warn(ctrl->device, "async event result %08x\n", result);
4302 }
Christoph Hellwig868c2392018-05-22 11:09:54 +02004303}
4304
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004305void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004306 volatile union nvme_result *res)
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004307{
4308 u32 result = le32_to_cpu(res->u32);
Chaitanya Kulkarni09bd1ff2018-09-17 10:47:06 -07004309 u32 aer_type = result & 0x07;
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004310
4311 if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
4312 return;
4313
Chaitanya Kulkarni09bd1ff2018-09-17 10:47:06 -07004314 switch (aer_type) {
Christoph Hellwig868c2392018-05-22 11:09:54 +02004315 case NVME_AER_NOTICE:
4316 nvme_handle_aen_notice(ctrl, result);
4317 break;
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004318 case NVME_AER_ERROR:
4319 case NVME_AER_SMART:
4320 case NVME_AER_CSS:
4321 case NVME_AER_VS:
Chaitanya Kulkarni09bd1ff2018-09-17 10:47:06 -07004322 trace_nvme_async_event(ctrl, aer_type);
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004323 ctrl->aen_result = result;
4324 break;
4325 default:
4326 break;
4327 }
Sagi Grimbergc669ccd2017-05-04 13:33:14 +03004328 queue_work(nvme_wq, &ctrl->async_event_work);
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004329}
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004330EXPORT_SYMBOL_GPL(nvme_complete_async_event);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004331
Sagi Grimbergd09f2b42017-07-02 10:56:43 +03004332void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
Ming Lin576d55d2016-02-10 10:03:32 -08004333{
Christoph Hellwig0d0b6602018-05-14 08:48:54 +02004334 nvme_mpath_stop(ctrl);
Sagi Grimbergd09f2b42017-07-02 10:56:43 +03004335 nvme_stop_keep_alive(ctrl);
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004336 flush_work(&ctrl->async_event_work);
Arnav Dawnb6dccf72017-07-12 16:10:40 +05304337 cancel_work_sync(&ctrl->fw_act_work);
Sagi Grimbergd09f2b42017-07-02 10:56:43 +03004338}
4339EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
Christoph Hellwig5955be22016-04-26 13:51:59 +02004340
Sagi Grimbergd09f2b42017-07-02 10:56:43 +03004341void nvme_start_ctrl(struct nvme_ctrl *ctrl)
4342{
Baolin Wang58874502020-07-13 14:25:21 +08004343 nvme_start_keep_alive(ctrl);
Sagi Grimbergd09f2b42017-07-02 10:56:43 +03004344
Sagi Grimberg93da4022019-08-22 11:25:46 -07004345 nvme_enable_aen(ctrl);
4346
Sagi Grimbergd09f2b42017-07-02 10:56:43 +03004347 if (ctrl->queue_count > 1) {
4348 nvme_queue_scan(ctrl);
Sagi Grimbergd09f2b42017-07-02 10:56:43 +03004349 nvme_start_queues(ctrl);
4350 }
Israel Rukshince151812020-03-24 17:29:43 +02004351 ctrl->created = true;
Sagi Grimbergd09f2b42017-07-02 10:56:43 +03004352}
4353EXPORT_SYMBOL_GPL(nvme_start_ctrl);
4354
4355void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
4356{
Akinobu Mitaf79d5fd2019-06-09 23:17:01 +09004357 nvme_fault_inject_fini(&ctrl->fault_inject);
Yufen Yu510a4052019-05-16 19:30:07 -07004358 dev_pm_qos_hide_latency_tolerance(ctrl->device);
Christoph Hellwiga6a51492017-10-18 16:59:25 +02004359 cdev_device_del(&ctrl->cdev, ctrl->device);
Israel Rukshin726612b2020-03-24 17:29:42 +02004360 nvme_put_ctrl(ctrl);
Keith Busch53029b02015-11-28 15:41:02 +01004361}
Ming Lin576d55d2016-02-10 10:03:32 -08004362EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
Keith Busch53029b02015-11-28 15:41:02 +01004363
Christoph Hellwigd22524a2017-10-18 13:25:42 +02004364static void nvme_free_ctrl(struct device *dev)
Keith Busch53029b02015-11-28 15:41:02 +01004365{
Christoph Hellwigd22524a2017-10-18 13:25:42 +02004366 struct nvme_ctrl *ctrl =
4367 container_of(dev, struct nvme_ctrl, ctrl_device);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01004368 struct nvme_subsystem *subsys = ctrl->subsys;
Keith Buschbe93e872020-06-29 12:06:40 -07004369 struct nvme_cel *cel, *next;
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004370
Keith Busch733e4b62019-09-05 10:33:54 -06004371 if (subsys && ctrl->instance != subsys->instance)
4372 ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4373
Keith Buschbe93e872020-06-29 12:06:40 -07004374 list_for_each_entry_safe(cel, next, &ctrl->cels, entry) {
4375 list_del(&cel->entry);
4376 kfree(cel);
4377 }
4378
Christoph Hellwig0d0b6602018-05-14 08:48:54 +02004379 nvme_mpath_uninit(ctrl);
Sagi Grimberg092ff052018-12-13 12:34:07 -08004380 __free_page(ctrl->discard_page);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004381
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01004382 if (subsys) {
Christoph Hellwig32fd90c2019-05-08 09:48:27 +02004383 mutex_lock(&nvme_subsystems_lock);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01004384 list_del(&ctrl->subsys_entry);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01004385 sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
Christoph Hellwig32fd90c2019-05-08 09:48:27 +02004386 mutex_unlock(&nvme_subsystems_lock);
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01004387 }
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004388
4389 ctrl->ops->free_ctrl(ctrl);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004390
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01004391 if (subsys)
4392 nvme_put_subsystem(subsys);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004393}
4394
4395/*
4396 * Initialize a NVMe controller structures. This needs to be called during
4397 * earliest initialization so that we have the initialized structured around
4398 * during probing.
4399 */
4400int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
4401 const struct nvme_ctrl_ops *ops, unsigned long quirks)
4402{
4403 int ret;
4404
Christoph Hellwigbb8d2612016-04-26 13:51:57 +02004405 ctrl->state = NVME_CTRL_NEW;
4406 spin_lock_init(&ctrl->lock);
Keith Busche7ad43c2019-01-28 09:46:07 -07004407 mutex_init(&ctrl->scan_lock);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004408 INIT_LIST_HEAD(&ctrl->namespaces);
Keith Buschbe93e872020-06-29 12:06:40 -07004409 INIT_LIST_HEAD(&ctrl->cels);
Jianchao Wang765cc0312018-02-12 20:54:46 +08004410 init_rwsem(&ctrl->namespaces_rwsem);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004411 ctrl->dev = dev;
4412 ctrl->ops = ops;
4413 ctrl->quirks = quirks;
Max Gurtovoy4fea2432020-06-16 12:34:21 +03004414 ctrl->numa_node = NUMA_NO_NODE;
Christoph Hellwig5955be22016-04-26 13:51:59 +02004415 INIT_WORK(&ctrl->scan_work, nvme_scan_work);
Christoph Hellwigf866fc422016-04-26 13:52:00 +02004416 INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
Arnav Dawnb6dccf72017-07-12 16:10:40 +05304417 INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
Christoph Hellwigc5017e82017-10-29 10:44:29 +02004418 INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
Keith Buschc1ac9a4b2019-09-04 10:06:11 -06004419 init_waitqueue_head(&ctrl->state_wq);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004420
James Smart230f1f92018-06-12 16:28:24 -07004421 INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
4422 memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
4423 ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
4424
Jens Axboecb5b7262018-12-12 09:18:11 -07004425 BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
4426 PAGE_SIZE);
4427 ctrl->discard_page = alloc_page(GFP_KERNEL);
4428 if (!ctrl->discard_page) {
4429 ret = -ENOMEM;
4430 goto out;
4431 }
4432
Christoph Hellwig9843f682017-10-18 13:10:01 +02004433 ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL);
4434 if (ret < 0)
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004435 goto out;
Christoph Hellwig9843f682017-10-18 13:10:01 +02004436 ctrl->instance = ret;
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004437
Christoph Hellwigd22524a2017-10-18 13:25:42 +02004438 device_initialize(&ctrl->ctrl_device);
4439 ctrl->device = &ctrl->ctrl_device;
Christoph Hellwiga6a51492017-10-18 16:59:25 +02004440 ctrl->device->devt = MKDEV(MAJOR(nvme_chr_devt), ctrl->instance);
Christoph Hellwigd22524a2017-10-18 13:25:42 +02004441 ctrl->device->class = nvme_class;
4442 ctrl->device->parent = ctrl->dev;
4443 ctrl->device->groups = nvme_dev_attr_groups;
4444 ctrl->device->release = nvme_free_ctrl;
4445 dev_set_drvdata(ctrl->device, ctrl);
4446 ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
4447 if (ret)
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004448 goto out_release_instance;
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004449
Israel Rukshinb780d742020-03-24 17:29:41 +02004450 nvme_get_ctrl(ctrl);
Christoph Hellwiga6a51492017-10-18 16:59:25 +02004451 cdev_init(&ctrl->cdev, &nvme_dev_fops);
4452 ctrl->cdev.owner = ops->module;
4453 ret = cdev_device_add(&ctrl->cdev, ctrl->device);
Christoph Hellwigd22524a2017-10-18 13:25:42 +02004454 if (ret)
4455 goto out_free_name;
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004456
Andy Lutomirskic5552fd2017-02-07 10:08:45 -08004457 /*
4458 * Initialize latency tolerance controls. The sysfs files won't
4459 * be visible to userspace unless the device actually supports APST.
4460 */
4461 ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
4462 dev_pm_qos_update_user_latency_tolerance(ctrl->device,
4463 min(default_ps_max_latency_us, (unsigned long)S32_MAX));
4464
Akinobu Mitaf79d5fd2019-06-09 23:17:01 +09004465 nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
4466
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004467 return 0;
Christoph Hellwigd22524a2017-10-18 13:25:42 +02004468out_free_name:
Israel Rukshinb780d742020-03-24 17:29:41 +02004469 nvme_put_ctrl(ctrl);
Keith Buschd6a2b9532018-11-26 16:39:47 -07004470 kfree_const(ctrl->device->kobj.name);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004471out_release_instance:
Christoph Hellwig9843f682017-10-18 13:10:01 +02004472 ida_simple_remove(&nvme_instance_ida, ctrl->instance);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004473out:
Jens Axboecb5b7262018-12-12 09:18:11 -07004474 if (ctrl->discard_page)
4475 __free_page(ctrl->discard_page);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004476 return ret;
4477}
Ming Lin576d55d2016-02-10 10:03:32 -08004478EXPORT_SYMBOL_GPL(nvme_init_ctrl);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004479
Keith Busch69d9a992016-02-24 09:15:56 -07004480/**
4481 * nvme_kill_queues(): Ends all namespace queues
4482 * @ctrl: the dead controller that needs to end
4483 *
4484 * Call this function when the driver determines it is unable to get the
4485 * controller in a state capable of servicing IO.
4486 */
4487void nvme_kill_queues(struct nvme_ctrl *ctrl)
4488{
4489 struct nvme_ns *ns;
4490
Jianchao Wang765cc0312018-02-12 20:54:46 +08004491 down_read(&ctrl->namespaces_rwsem);
Ming Lei82654b62017-06-02 16:32:08 +08004492
Ming Lei443bd902017-06-19 10:21:08 +08004493 /* Forcibly unquiesce queues to avoid blocking dispatch */
Igor Konopko751a0cc2018-11-23 16:58:10 +01004494 if (ctrl->admin_q && !blk_queue_dying(ctrl->admin_q))
Scott Bauer7dd1ab12017-07-25 10:27:06 -06004495 blk_mq_unquiesce_queue(ctrl->admin_q);
Ming Lei443bd902017-06-19 10:21:08 +08004496
Scott Bauercf39a6b2018-06-29 13:03:28 -06004497 list_for_each_entry(ns, &ctrl->namespaces, list)
4498 nvme_set_queue_dying(ns);
Ming Lei806f0262017-05-22 23:05:03 +08004499
Jianchao Wang765cc0312018-02-12 20:54:46 +08004500 up_read(&ctrl->namespaces_rwsem);
Keith Busch69d9a992016-02-24 09:15:56 -07004501}
Linus Torvalds237045f2016-03-18 17:13:31 -07004502EXPORT_SYMBOL_GPL(nvme_kill_queues);
Keith Busch69d9a992016-02-24 09:15:56 -07004503
Keith Busch302ad8c2017-03-01 14:22:12 -05004504void nvme_unfreeze(struct nvme_ctrl *ctrl)
4505{
4506 struct nvme_ns *ns;
4507
Jianchao Wang765cc0312018-02-12 20:54:46 +08004508 down_read(&ctrl->namespaces_rwsem);
Keith Busch302ad8c2017-03-01 14:22:12 -05004509 list_for_each_entry(ns, &ctrl->namespaces, list)
4510 blk_mq_unfreeze_queue(ns->queue);
Jianchao Wang765cc0312018-02-12 20:54:46 +08004511 up_read(&ctrl->namespaces_rwsem);
Keith Busch302ad8c2017-03-01 14:22:12 -05004512}
4513EXPORT_SYMBOL_GPL(nvme_unfreeze);
4514
4515void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
4516{
4517 struct nvme_ns *ns;
4518
Jianchao Wang765cc0312018-02-12 20:54:46 +08004519 down_read(&ctrl->namespaces_rwsem);
Keith Busch302ad8c2017-03-01 14:22:12 -05004520 list_for_each_entry(ns, &ctrl->namespaces, list) {
4521 timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
4522 if (timeout <= 0)
4523 break;
4524 }
Jianchao Wang765cc0312018-02-12 20:54:46 +08004525 up_read(&ctrl->namespaces_rwsem);
Keith Busch302ad8c2017-03-01 14:22:12 -05004526}
4527EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
4528
4529void nvme_wait_freeze(struct nvme_ctrl *ctrl)
4530{
4531 struct nvme_ns *ns;
4532
Jianchao Wang765cc0312018-02-12 20:54:46 +08004533 down_read(&ctrl->namespaces_rwsem);
Keith Busch302ad8c2017-03-01 14:22:12 -05004534 list_for_each_entry(ns, &ctrl->namespaces, list)
4535 blk_mq_freeze_queue_wait(ns->queue);
Jianchao Wang765cc0312018-02-12 20:54:46 +08004536 up_read(&ctrl->namespaces_rwsem);
Keith Busch302ad8c2017-03-01 14:22:12 -05004537}
4538EXPORT_SYMBOL_GPL(nvme_wait_freeze);
4539
4540void nvme_start_freeze(struct nvme_ctrl *ctrl)
4541{
4542 struct nvme_ns *ns;
4543
Jianchao Wang765cc0312018-02-12 20:54:46 +08004544 down_read(&ctrl->namespaces_rwsem);
Keith Busch302ad8c2017-03-01 14:22:12 -05004545 list_for_each_entry(ns, &ctrl->namespaces, list)
Ming Lei1671d522017-03-27 20:06:57 +08004546 blk_freeze_queue_start(ns->queue);
Jianchao Wang765cc0312018-02-12 20:54:46 +08004547 up_read(&ctrl->namespaces_rwsem);
Keith Busch302ad8c2017-03-01 14:22:12 -05004548}
4549EXPORT_SYMBOL_GPL(nvme_start_freeze);
4550
Keith Busch25646262016-01-04 09:10:57 -07004551void nvme_stop_queues(struct nvme_ctrl *ctrl)
Sagi Grimberg363c9aa2015-12-24 15:26:59 +01004552{
4553 struct nvme_ns *ns;
4554
Jianchao Wang765cc0312018-02-12 20:54:46 +08004555 down_read(&ctrl->namespaces_rwsem);
Bart Van Asschea6eaa882016-10-28 17:23:40 -07004556 list_for_each_entry(ns, &ctrl->namespaces, list)
Bart Van Assche3174dd32016-10-28 17:23:19 -07004557 blk_mq_quiesce_queue(ns->queue);
Jianchao Wang765cc0312018-02-12 20:54:46 +08004558 up_read(&ctrl->namespaces_rwsem);
Sagi Grimberg363c9aa2015-12-24 15:26:59 +01004559}
Ming Lin576d55d2016-02-10 10:03:32 -08004560EXPORT_SYMBOL_GPL(nvme_stop_queues);
Sagi Grimberg363c9aa2015-12-24 15:26:59 +01004561
Keith Busch25646262016-01-04 09:10:57 -07004562void nvme_start_queues(struct nvme_ctrl *ctrl)
Sagi Grimberg363c9aa2015-12-24 15:26:59 +01004563{
4564 struct nvme_ns *ns;
4565
Jianchao Wang765cc0312018-02-12 20:54:46 +08004566 down_read(&ctrl->namespaces_rwsem);
Sagi Grimberg8d7b8fa2017-07-04 18:16:58 +03004567 list_for_each_entry(ns, &ctrl->namespaces, list)
Ming Leif6601742017-06-06 23:22:04 +08004568 blk_mq_unquiesce_queue(ns->queue);
Jianchao Wang765cc0312018-02-12 20:54:46 +08004569 up_read(&ctrl->namespaces_rwsem);
Sagi Grimberg363c9aa2015-12-24 15:26:59 +01004570}
Ming Lin576d55d2016-02-10 10:03:32 -08004571EXPORT_SYMBOL_GPL(nvme_start_queues);
Sagi Grimberg363c9aa2015-12-24 15:26:59 +01004572
Keith Buschd6135c3a2019-05-14 14:46:09 -06004573
4574void nvme_sync_queues(struct nvme_ctrl *ctrl)
4575{
4576 struct nvme_ns *ns;
4577
4578 down_read(&ctrl->namespaces_rwsem);
4579 list_for_each_entry(ns, &ctrl->namespaces, list)
4580 blk_sync_queue(ns->queue);
4581 up_read(&ctrl->namespaces_rwsem);
Edmund Nadolski03894b7a2019-09-03 14:08:47 -06004582
4583 if (ctrl->admin_q)
4584 blk_sync_queue(ctrl->admin_q);
Keith Buschd6135c3a2019-05-14 14:46:09 -06004585}
4586EXPORT_SYMBOL_GPL(nvme_sync_queues);
4587
Logan Gunthorpef783f442020-07-24 11:25:15 -06004588struct nvme_ctrl *nvme_ctrl_get_by_path(const char *path)
4589{
4590 struct nvme_ctrl *ctrl;
4591 struct file *f;
4592
4593 f = filp_open(path, O_RDWR, 0);
4594 if (IS_ERR(f))
4595 return ERR_CAST(f);
4596
4597 if (f->f_op != &nvme_dev_fops) {
4598 ctrl = ERR_PTR(-EINVAL);
4599 goto out_close;
4600 }
4601
4602 ctrl = f->private_data;
4603 nvme_get_ctrl(ctrl);
4604
4605out_close:
4606 filp_close(f, NULL);
4607 return ctrl;
4608}
4609EXPORT_SYMBOL_NS_GPL(nvme_ctrl_get_by_path, NVME_TARGET_PASSTHRU);
4610
Christoph Hellwig81101542019-04-30 11:36:52 -04004611/*
4612 * Check we didn't inadvertently grow the command structure sizes:
4613 */
4614static inline void _nvme_check_size(void)
4615{
4616 BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
4617 BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
4618 BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
4619 BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
4620 BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
4621 BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
4622 BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
4623 BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
4624 BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
4625 BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
4626 BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
4627 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
4628 BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
Keith Busch240e6ee2020-06-29 12:06:41 -07004629 BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
4630 BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
Christoph Hellwig81101542019-04-30 11:36:52 -04004631 BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
4632 BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
4633 BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
4634 BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
4635}
4636
4637
Christoph Hellwig893a74b2019-04-30 11:37:43 -04004638static int __init nvme_core_init(void)
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004639{
Roy Shtermanb227c592018-01-14 12:39:02 +02004640 int result = -ENOMEM;
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004641
Christoph Hellwig81101542019-04-30 11:36:52 -04004642 _nvme_check_size();
4643
Sagi Grimberg9a6327d2017-06-07 20:31:55 +02004644 nvme_wq = alloc_workqueue("nvme-wq",
4645 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4646 if (!nvme_wq)
Roy Shtermanb227c592018-01-14 12:39:02 +02004647 goto out;
4648
4649 nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
4650 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4651 if (!nvme_reset_wq)
4652 goto destroy_wq;
4653
4654 nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
4655 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4656 if (!nvme_delete_wq)
4657 goto destroy_reset_wq;
Sagi Grimberg9a6327d2017-06-07 20:31:55 +02004658
Christoph Hellwiga6a51492017-10-18 16:59:25 +02004659 result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme");
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004660 if (result < 0)
Roy Shtermanb227c592018-01-14 12:39:02 +02004661 goto destroy_delete_wq;
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004662
4663 nvme_class = class_create(THIS_MODULE, "nvme");
4664 if (IS_ERR(nvme_class)) {
4665 result = PTR_ERR(nvme_class);
4666 goto unregister_chrdev;
4667 }
Sagi Grimberga42f42e2019-09-04 14:29:48 -07004668 nvme_class->dev_uevent = nvme_class_uevent;
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004669
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01004670 nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
4671 if (IS_ERR(nvme_subsys_class)) {
4672 result = PTR_ERR(nvme_subsys_class);
4673 goto destroy_class;
4674 }
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004675 return 0;
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004676
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01004677destroy_class:
4678 class_destroy(nvme_class);
Sagi Grimberg9a6327d2017-06-07 20:31:55 +02004679unregister_chrdev:
Christoph Hellwiga6a51492017-10-18 16:59:25 +02004680 unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
Roy Shtermanb227c592018-01-14 12:39:02 +02004681destroy_delete_wq:
4682 destroy_workqueue(nvme_delete_wq);
4683destroy_reset_wq:
4684 destroy_workqueue(nvme_reset_wq);
Sagi Grimberg9a6327d2017-06-07 20:31:55 +02004685destroy_wq:
4686 destroy_workqueue(nvme_wq);
Roy Shtermanb227c592018-01-14 12:39:02 +02004687out:
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004688 return result;
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004689}
4690
Christoph Hellwig893a74b2019-04-30 11:37:43 -04004691static void __exit nvme_core_exit(void)
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004692{
Christoph Hellwigab9e00c2017-11-09 13:48:55 +01004693 class_destroy(nvme_subsys_class);
Christoph Hellwigf3ca80f2015-11-28 15:40:19 +01004694 class_destroy(nvme_class);
Christoph Hellwiga6a51492017-10-18 16:59:25 +02004695 unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
Roy Shtermanb227c592018-01-14 12:39:02 +02004696 destroy_workqueue(nvme_delete_wq);
4697 destroy_workqueue(nvme_reset_wq);
Sagi Grimberg9a6327d2017-06-07 20:31:55 +02004698 destroy_workqueue(nvme_wq);
Max Gurtovoyf41cfd52020-03-18 17:27:59 +02004699 ida_destroy(&nvme_instance_ida);
Christoph Hellwig5bae7f72015-11-28 15:39:07 +01004700}
Ming Lin576d55d2016-02-10 10:03:32 -08004701
4702MODULE_LICENSE("GPL");
4703MODULE_VERSION("1.0");
4704module_init(nvme_core_init);
4705module_exit(nvme_core_exit);