blob: a35e12f8e68baa8b1baf45d5234dff62af91813e [file] [log] [blame]
Bean Huo67351112020-06-05 22:05:19 +02001/* SPDX-License-Identifier: GPL-2.0-or-later */
Vinayak Holikattie0eca632013-02-25 21:44:33 +05302/*
3 * Universal Flash Storage Host controller driver
Vinayak Holikattie0eca632013-02-25 21:44:33 +05304 * Copyright (C) 2011-2013 Samsung India Software Operations
Yaniv Gardidc3c8d32016-02-01 15:02:46 +02005 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
Vinayak Holikattie0eca632013-02-25 21:44:33 +05306 *
7 * Authors:
8 * Santosh Yaraganavi <santosh.sy@samsung.com>
9 * Vinayak Holikatti <h.vinayak@samsung.com>
Vinayak Holikattie0eca632013-02-25 21:44:33 +053010 */
11
12#ifndef _UFSHCD_H
13#define _UFSHCD_H
14
Stanley Chu5a244e02020-01-29 18:52:50 +080015#include <linux/bitfield.h>
Eric Biggers1e8d44b2021-10-18 11:04:51 -070016#include <linux/blk-crypto-profile.h>
Bart Van Assche3f06f782022-04-19 15:58:08 -070017#include <linux/blk-mq.h>
18#include <linux/devfreq.h>
Akinobu Mita045da302023-11-18 21:44:43 +090019#include <linux/fault-inject.h>
Can Guoe02288e2022-12-14 19:06:21 -080020#include <linux/msi.h>
Bart Van Assche3f06f782022-04-19 15:58:08 -070021#include <linux/pm_runtime.h>
Bean Huof3e57da2023-01-08 23:40:56 +010022#include <linux/dma-direction.h>
Bart Van Assche3f06f782022-04-19 15:58:08 -070023#include <scsi/scsi_device.h>
Bart Van Asschecce9fd62023-07-27 12:41:17 -070024#include <scsi/scsi_host.h>
Bart Van Asschedd113762022-05-11 14:25:52 -070025#include <ufs/unipro.h>
26#include <ufs/ufs.h>
27#include <ufs/ufs_quirks.h>
28#include <ufs/ufshci.h>
Vinayak Holikattie0eca632013-02-25 21:44:33 +053029
30#define UFSHCD "ufshcd"
Vinayak Holikattie0eca632013-02-25 21:44:33 +053031
Bart Van Assche858231b2023-09-21 12:22:47 -070032struct scsi_device;
Sujit Reddy Thumma5c0c28a2014-09-25 15:32:21 +030033struct ufs_hba;
34
Sujit Reddy Thumma5a0b0cb2013-07-30 00:35:57 +053035enum dev_cmd_type {
36 DEV_CMD_TYPE_NOP = 0x0,
Dolev Raviv68078d52013-07-30 00:35:58 +053037 DEV_CMD_TYPE_QUERY = 0x1,
Bean Huo6ff265fc2022-12-01 15:04:37 +010038 DEV_CMD_TYPE_RPMB = 0x2,
Sujit Reddy Thumma5a0b0cb2013-07-30 00:35:57 +053039};
40
Stanley Chue965e5e2020-12-05 19:58:59 +080041enum ufs_event_type {
42 /* uic specific errors */
43 UFS_EVT_PA_ERR = 0,
44 UFS_EVT_DL_ERR,
45 UFS_EVT_NL_ERR,
46 UFS_EVT_TL_ERR,
47 UFS_EVT_DME_ERR,
48
49 /* fatal errors */
50 UFS_EVT_AUTO_HIBERN8_ERR,
51 UFS_EVT_FATAL_ERR,
52 UFS_EVT_LINK_STARTUP_FAIL,
53 UFS_EVT_RESUME_ERR,
54 UFS_EVT_SUSPEND_ERR,
Asutosh Dasb294ff32021-04-23 17:20:16 -070055 UFS_EVT_WL_SUSP_ERR,
56 UFS_EVT_WL_RES_ERR,
Stanley Chue965e5e2020-12-05 19:58:59 +080057
58 /* abnormal events */
59 UFS_EVT_DEV_RESET,
60 UFS_EVT_HOST_RESET,
61 UFS_EVT_ABORT,
62
63 UFS_EVT_CNT,
64};
65
Vinayak Holikattie0eca632013-02-25 21:44:33 +053066/**
67 * struct uic_command - UIC command structure
68 * @command: UIC command
69 * @argument1: UIC command argument 1
70 * @argument2: UIC command argument 2
71 * @argument3: UIC command argument 3
Can Guo0f52fcb92020-11-02 22:24:40 -080072 * @cmd_active: Indicate if UIC command is outstanding
Seungwon Jeon6ccf44fe2013-06-26 22:39:29 +053073 * @done: UIC command completion
Vinayak Holikattie0eca632013-02-25 21:44:33 +053074 */
75struct uic_command {
76 u32 command;
77 u32 argument1;
78 u32 argument2;
79 u32 argument3;
Can Guo0f52fcb92020-11-02 22:24:40 -080080 int cmd_active;
Seungwon Jeon6ccf44fe2013-06-26 22:39:29 +053081 struct completion done;
Vinayak Holikattie0eca632013-02-25 21:44:33 +053082};
83
Subhash Jadavani57d104c2014-09-25 15:32:30 +030084/* Used to differentiate the power management options */
85enum ufs_pm_op {
86 UFS_RUNTIME_PM,
87 UFS_SYSTEM_PM,
88 UFS_SHUTDOWN_PM,
89};
90
Subhash Jadavani57d104c2014-09-25 15:32:30 +030091/* Host <-> Device UniPro Link state */
92enum uic_link_state {
93 UIC_LINK_OFF_STATE = 0, /* Link powered down or disabled */
94 UIC_LINK_ACTIVE_STATE = 1, /* Link is in Fast/Slow/Sleep state */
95 UIC_LINK_HIBERN8_STATE = 2, /* Link is in Hibernate state */
Can Guo4db7a232020-08-09 05:15:51 -070096 UIC_LINK_BROKEN_STATE = 3, /* Link is in broken state */
Subhash Jadavani57d104c2014-09-25 15:32:30 +030097};
98
99#define ufshcd_is_link_off(hba) ((hba)->uic_link_state == UIC_LINK_OFF_STATE)
100#define ufshcd_is_link_active(hba) ((hba)->uic_link_state == \
101 UIC_LINK_ACTIVE_STATE)
102#define ufshcd_is_link_hibern8(hba) ((hba)->uic_link_state == \
103 UIC_LINK_HIBERN8_STATE)
Can Guo4db7a232020-08-09 05:15:51 -0700104#define ufshcd_is_link_broken(hba) ((hba)->uic_link_state == \
105 UIC_LINK_BROKEN_STATE)
Subhash Jadavani57d104c2014-09-25 15:32:30 +0300106#define ufshcd_set_link_off(hba) ((hba)->uic_link_state = UIC_LINK_OFF_STATE)
107#define ufshcd_set_link_active(hba) ((hba)->uic_link_state = \
108 UIC_LINK_ACTIVE_STATE)
109#define ufshcd_set_link_hibern8(hba) ((hba)->uic_link_state = \
110 UIC_LINK_HIBERN8_STATE)
Can Guo4db7a232020-08-09 05:15:51 -0700111#define ufshcd_set_link_broken(hba) ((hba)->uic_link_state = \
112 UIC_LINK_BROKEN_STATE)
Subhash Jadavani57d104c2014-09-25 15:32:30 +0300113
Stanley Chu1764fa22020-03-27 17:58:35 +0800114#define ufshcd_set_ufs_dev_active(h) \
115 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
116#define ufshcd_set_ufs_dev_sleep(h) \
117 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
118#define ufshcd_set_ufs_dev_poweroff(h) \
119 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
Adrian Hunterfe1d4c22020-11-03 16:14:02 +0200120#define ufshcd_set_ufs_dev_deepsleep(h) \
121 ((h)->curr_dev_pwr_mode = UFS_DEEPSLEEP_PWR_MODE)
Stanley Chu1764fa22020-03-27 17:58:35 +0800122#define ufshcd_is_ufs_dev_active(h) \
123 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
124#define ufshcd_is_ufs_dev_sleep(h) \
125 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
126#define ufshcd_is_ufs_dev_poweroff(h) \
127 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
Adrian Hunterfe1d4c22020-11-03 16:14:02 +0200128#define ufshcd_is_ufs_dev_deepsleep(h) \
129 ((h)->curr_dev_pwr_mode == UFS_DEEPSLEEP_PWR_MODE)
Stanley Chu1764fa22020-03-27 17:58:35 +0800130
Subhash Jadavani57d104c2014-09-25 15:32:30 +0300131/*
132 * UFS Power management levels.
Adrian Hunterfe1d4c22020-11-03 16:14:02 +0200133 * Each level is in increasing order of power savings, except DeepSleep
134 * which is lower than PowerDown with power on but not PowerDown with
135 * power off.
Subhash Jadavani57d104c2014-09-25 15:32:30 +0300136 */
137enum ufs_pm_level {
Bart Van Asschee2ac7ab2021-05-19 13:20:58 -0700138 UFS_PM_LVL_0,
139 UFS_PM_LVL_1,
140 UFS_PM_LVL_2,
141 UFS_PM_LVL_3,
142 UFS_PM_LVL_4,
143 UFS_PM_LVL_5,
144 UFS_PM_LVL_6,
Subhash Jadavani57d104c2014-09-25 15:32:30 +0300145 UFS_PM_LVL_MAX
146};
147
148struct ufs_pm_lvl_states {
149 enum ufs_dev_pwr_mode dev_state;
150 enum uic_link_state link_state;
151};
152
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530153/**
154 * struct ufshcd_lrb - local reference block
155 * @utr_descriptor_ptr: UTRD address of the command
Sujit Reddy Thumma5a0b0cb2013-07-30 00:35:57 +0530156 * @ucd_req_ptr: UCD address of the command
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530157 * @ucd_rsp_ptr: Response UPIU address for this command
158 * @ucd_prdt_ptr: PRDT address of the command
Dolev Ravivff8e20c2016-12-22 18:42:18 -0800159 * @utrd_dma_addr: UTRD dma address for debug
160 * @ucd_prdt_dma_addr: PRDT dma address for debug
161 * @ucd_rsp_dma_addr: UPIU response dma address for debug
162 * @ucd_req_dma_addr: UPIU request dma address for debug
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530163 * @cmd: pointer to SCSI command
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530164 * @scsi_status: SCSI status of the command
165 * @command_type: SCSI, UFS, Query.
166 * @task_tag: Task tag of the command
167 * @lun: LUN of the command
Sujit Reddy Thumma5a0b0cb2013-07-30 00:35:57 +0530168 * @intr_cmd: Interrupt command (doesn't participate in interrupt aggregation)
Daniil Lunev0f85e742022-08-04 06:50:34 +1000169 * @issue_time_stamp: time stamp for debug purposes (CLOCK_MONOTONIC)
170 * @issue_time_stamp_local_clock: time stamp for debug purposes (local_clock)
171 * @compl_time_stamp: time stamp for statistics (CLOCK_MONOTONIC)
172 * @compl_time_stamp_local_clock: time stamp for debug purposes (local_clock)
Satya Tangiraladf043c742020-07-06 20:04:14 +0000173 * @crypto_key_slot: the key slot to use for inline crypto (-1 if none)
174 * @data_unit_num: the data unit number for the first block for inline crypto
Gilad Bronere0b299e2017-02-03 16:56:40 -0800175 * @req_abort_skip: skip request abort task flag
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530176 */
177struct ufshcd_lrb {
178 struct utp_transfer_req_desc *utr_descriptor_ptr;
Sujit Reddy Thumma5a0b0cb2013-07-30 00:35:57 +0530179 struct utp_upiu_req *ucd_req_ptr;
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530180 struct utp_upiu_rsp *ucd_rsp_ptr;
181 struct ufshcd_sg_entry *ucd_prdt_ptr;
182
Dolev Ravivff8e20c2016-12-22 18:42:18 -0800183 dma_addr_t utrd_dma_addr;
184 dma_addr_t ucd_req_dma_addr;
185 dma_addr_t ucd_rsp_dma_addr;
186 dma_addr_t ucd_prdt_dma_addr;
187
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530188 struct scsi_cmnd *cmd;
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530189 int scsi_status;
190
191 int command_type;
192 int task_tag;
Subhash Jadavani0ce147d2014-09-25 15:32:29 +0300193 u8 lun; /* UPIU LUN id field is only 8-bit wide */
Sujit Reddy Thumma5a0b0cb2013-07-30 00:35:57 +0530194 bool intr_cmd;
Dolev Ravivff8e20c2016-12-22 18:42:18 -0800195 ktime_t issue_time_stamp;
Daniil Lunev0f85e742022-08-04 06:50:34 +1000196 u64 issue_time_stamp_local_clock;
Zang Leigang09017182017-09-27 10:06:06 +0800197 ktime_t compl_time_stamp;
Daniil Lunev0f85e742022-08-04 06:50:34 +1000198 u64 compl_time_stamp_local_clock;
Satya Tangiraladf043c742020-07-06 20:04:14 +0000199#ifdef CONFIG_SCSI_UFS_CRYPTO
200 int crypto_key_slot;
201 u64 data_unit_num;
202#endif
Gilad Bronere0b299e2017-02-03 16:56:40 -0800203
204 bool req_abort_skip;
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530205};
206
Sujit Reddy Thumma5a0b0cb2013-07-30 00:35:57 +0530207/**
Bart Van Asschee2566e02023-07-27 12:41:22 -0700208 * struct ufs_query_req - parameters for building a query request
209 * @query_func: UPIU header query function
210 * @upiu_req: the query request data
211 */
212struct ufs_query_req {
213 u8 query_func;
214 struct utp_upiu_query upiu_req;
215};
216
217/**
218 * struct ufs_query_resp - UPIU QUERY
219 * @response: device response code
220 * @upiu_res: query response data
221 */
222struct ufs_query_res {
223 struct utp_upiu_query upiu_res;
224};
225
226/**
Tomas Winklera230c2f2016-02-09 10:25:41 +0200227 * struct ufs_query - holds relevant data structures for query request
Dolev Raviv68078d52013-07-30 00:35:58 +0530228 * @request: request upiu and function
229 * @descriptor: buffer for sending/receiving descriptor
230 * @response: response upiu and response
231 */
232struct ufs_query {
233 struct ufs_query_req request;
234 u8 *descriptor;
235 struct ufs_query_res response;
236};
237
238/**
Sujit Reddy Thumma5a0b0cb2013-07-30 00:35:57 +0530239 * struct ufs_dev_cmd - all assosiated fields with device management commands
240 * @type: device management command type - Query, NOP OUT
241 * @lock: lock to allow one command at a time
242 * @complete: internal commands completion
Bart Van Asschecff91da2022-04-19 15:58:07 -0700243 * @query: Device management query information
Sujit Reddy Thumma5a0b0cb2013-07-30 00:35:57 +0530244 */
245struct ufs_dev_cmd {
246 enum dev_cmd_type type;
247 struct mutex lock;
248 struct completion *complete;
Dolev Raviv68078d52013-07-30 00:35:58 +0530249 struct ufs_query query;
Sujit Reddy Thumma5a0b0cb2013-07-30 00:35:57 +0530250};
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530251
Sujit Reddy Thummac6e79da2014-09-25 15:32:23 +0300252/**
253 * struct ufs_clk_info - UFS clock related info
254 * @list: list headed by hba->clk_list_head
255 * @clk: clock node
256 * @name: clock name
257 * @max_freq: maximum frequency supported by the clock
Sahitya Tummala4cff6d992014-09-25 15:32:33 +0300258 * @min_freq: min frequency that can be used for clock scaling
Sahitya Tummala856b3482014-09-25 15:32:34 +0300259 * @curr_freq: indicates the current frequency that it is set to
Can Guo81309c22020-11-25 18:01:00 -0800260 * @keep_link_active: indicates that the clk should not be disabled if
Bart Van Asschecff91da2022-04-19 15:58:07 -0700261 * link is active
Sujit Reddy Thummac6e79da2014-09-25 15:32:23 +0300262 * @enabled: variable to check against multiple enable/disable
263 */
264struct ufs_clk_info {
265 struct list_head list;
266 struct clk *clk;
267 const char *name;
268 u32 max_freq;
Sahitya Tummala4cff6d992014-09-25 15:32:33 +0300269 u32 min_freq;
Sahitya Tummala856b3482014-09-25 15:32:34 +0300270 u32 curr_freq;
Can Guo81309c22020-11-25 18:01:00 -0800271 bool keep_link_active;
Sujit Reddy Thummac6e79da2014-09-25 15:32:23 +0300272 bool enabled;
273};
274
Yaniv Gardif06fcc72015-10-28 13:15:51 +0200275enum ufs_notify_change_status {
276 PRE_CHANGE,
277 POST_CHANGE,
278};
Dolev Raviv7eb584d2014-09-25 15:32:31 +0300279
280struct ufs_pa_layer_attr {
281 u32 gear_rx;
282 u32 gear_tx;
283 u32 lane_rx;
284 u32 lane_tx;
285 u32 pwr_rx;
286 u32 pwr_tx;
287 u32 hs_rate;
288};
289
290struct ufs_pwr_mode_info {
291 bool is_valid;
292 struct ufs_pa_layer_attr info;
293};
294
Sujit Reddy Thumma5c0c28a2014-09-25 15:32:21 +0300295/**
296 * struct ufs_hba_variant_ops - variant specific callbacks
297 * @name: variant name
298 * @init: called when the driver is initialized
299 * @exit: called to cleanup everything done in init
Yaniv Gardi9949e702015-05-17 18:55:05 +0300300 * @get_ufs_hci_version: called to get UFS HCI version
Sahitya Tummala856b3482014-09-25 15:32:34 +0300301 * @clk_scale_notify: notifies that clks are scaled up/down
Sujit Reddy Thumma5c0c28a2014-09-25 15:32:21 +0300302 * @setup_clocks: called before touching any of the controller registers
Sujit Reddy Thumma5c0c28a2014-09-25 15:32:21 +0300303 * @hce_enable_notify: called before and after HCE enable bit is set to allow
304 * variant specific Uni-Pro initialization.
305 * @link_startup_notify: called before and after Link startup is carried out
306 * to allow variant specific Uni-Pro initialization.
Dolev Raviv7eb584d2014-09-25 15:32:31 +0300307 * @pwr_change_notify: called before and after a power mode change
308 * is carried out to allow vendor spesific capabilities
309 * to be set.
Kiwoong Kim0e675ef2016-11-10 21:14:36 +0900310 * @setup_xfer_req: called before any transfer request is issued
311 * to set some things
Kiwoong Kimd2877be2016-11-10 21:16:15 +0900312 * @setup_task_mgmt: called before any task management request is issued
313 * to set some things
Kiwoong Kimee32c902016-11-10 21:17:43 +0900314 * @hibern8_notify: called around hibern8 enter/exit
Subhash Jadavani56d4a182016-12-05 19:25:32 -0800315 * @apply_dev_quirks: called to apply device specific quirks
Bart Van Asschecff91da2022-04-19 15:58:07 -0700316 * @fixup_dev_quirks: called to modify device specific quirks
Subhash Jadavani57d104c2014-09-25 15:32:30 +0300317 * @suspend: called during host controller PM callback
318 * @resume: called during host controller PM callback
Yaniv Gardi6e3fd442015-10-28 13:15:50 +0200319 * @dbg_register_dump: used to dump controller debug information
Joao Pinto4b9ffb52016-05-11 12:21:30 +0100320 * @phy_initialization: used to initialize phys
Bjorn Anderssond8d9f792019-08-28 12:17:54 -0700321 * @device_reset: called to issue a reset pulse on the UFS device
Bart Van Asschecff91da2022-04-19 15:58:07 -0700322 * @config_scaling_param: called to configure clock scaling parameters
Eric Biggers1bc726e2020-07-10 00:20:11 -0700323 * @program_key: program or evict an inline encryption key
Stanley Chu172614a2020-12-05 19:59:00 +0800324 * @event_notify: called to notify important events
Manivannan Sadhasivamc2c38c52022-12-22 19:39:56 +0530325 * @reinit_notify: called to notify reinit of UFSHCD during max gear switch
Asutosh Dasc263b4e2023-01-13 12:48:42 -0800326 * @mcq_config_resource: called to configure MCQ platform resources
Asutosh Das7224c802023-01-13 12:48:43 -0800327 * @get_hba_mac: called to get vendor specific mac value, mandatory for mcq mode
Asutosh Das2468da62023-01-13 12:48:45 -0800328 * @op_runtime_config: called to config Operation and runtime regs Pointers
Asutosh Dasf87b2c42023-01-13 12:48:50 -0800329 * @get_outstanding_cqs: called to get outstanding completion queues
Can Guoedb0db02022-12-14 19:06:20 -0800330 * @config_esi: called to config Event Specific Interrupt
Peter Wangdb06ae72024-03-01 11:46:10 +0800331 * @config_scsi_dev: called to configure SCSI device parameters
Sujit Reddy Thumma5c0c28a2014-09-25 15:32:21 +0300332 */
333struct ufs_hba_variant_ops {
334 const char *name;
335 int (*init)(struct ufs_hba *);
336 void (*exit)(struct ufs_hba *);
Yaniv Gardi9949e702015-05-17 18:55:05 +0300337 u32 (*get_ufs_hci_version)(struct ufs_hba *);
Yaniv Gardif06fcc72015-10-28 13:15:51 +0200338 int (*clk_scale_notify)(struct ufs_hba *, bool,
339 enum ufs_notify_change_status);
Subhash Jadavani1e879e82016-10-06 21:48:22 -0700340 int (*setup_clocks)(struct ufs_hba *, bool,
341 enum ufs_notify_change_status);
Yaniv Gardif06fcc72015-10-28 13:15:51 +0200342 int (*hce_enable_notify)(struct ufs_hba *,
343 enum ufs_notify_change_status);
344 int (*link_startup_notify)(struct ufs_hba *,
345 enum ufs_notify_change_status);
Dolev Raviv7eb584d2014-09-25 15:32:31 +0300346 int (*pwr_change_notify)(struct ufs_hba *,
Yaniv Gardif06fcc72015-10-28 13:15:51 +0200347 enum ufs_notify_change_status status,
348 struct ufs_pa_layer_attr *,
Dolev Raviv7eb584d2014-09-25 15:32:31 +0300349 struct ufs_pa_layer_attr *);
Bart Van Asscheb4276092021-12-03 15:19:35 -0800350 void (*setup_xfer_req)(struct ufs_hba *hba, int tag,
351 bool is_scsi_cmd);
Kiwoong Kimd2877be2016-11-10 21:16:15 +0900352 void (*setup_task_mgmt)(struct ufs_hba *, int, u8);
Kiwoong Kimee32c902016-11-10 21:17:43 +0900353 void (*hibern8_notify)(struct ufs_hba *, enum uic_cmd_dme,
Subhash Jadavani56d4a182016-12-05 19:25:32 -0800354 enum ufs_notify_change_status);
Bean Huo09750062020-01-20 14:08:14 +0100355 int (*apply_dev_quirks)(struct ufs_hba *hba);
Stanley Chuc28c00b2020-05-08 16:01:09 +0800356 void (*fixup_dev_quirks)(struct ufs_hba *hba);
Peter Wang9561f582021-10-06 13:47:05 +0800357 int (*suspend)(struct ufs_hba *, enum ufs_pm_op,
358 enum ufs_notify_change_status);
Subhash Jadavani57d104c2014-09-25 15:32:30 +0300359 int (*resume)(struct ufs_hba *, enum ufs_pm_op);
Yaniv Gardi6e3fd442015-10-28 13:15:50 +0200360 void (*dbg_register_dump)(struct ufs_hba *hba);
Joao Pinto4b9ffb52016-05-11 12:21:30 +0100361 int (*phy_initialization)(struct ufs_hba *);
Adrian Hunter151f1b62020-11-03 16:14:03 +0200362 int (*device_reset)(struct ufs_hba *hba);
Asutosh Das2c75f9a52020-03-25 11:29:01 -0700363 void (*config_scaling_param)(struct ufs_hba *hba,
Bart Van Asschec906e832022-04-19 15:57:57 -0700364 struct devfreq_dev_profile *profile,
365 struct devfreq_simple_ondemand_data *data);
Eric Biggers1bc726e2020-07-10 00:20:11 -0700366 int (*program_key)(struct ufs_hba *hba,
367 const union ufs_crypto_cfg_entry *cfg, int slot);
Stanley Chu172614a2020-12-05 19:59:00 +0800368 void (*event_notify)(struct ufs_hba *hba,
369 enum ufs_event_type evt, void *data);
Manivannan Sadhasivamc2c38c52022-12-22 19:39:56 +0530370 void (*reinit_notify)(struct ufs_hba *);
Asutosh Dasc263b4e2023-01-13 12:48:42 -0800371 int (*mcq_config_resource)(struct ufs_hba *hba);
Asutosh Das7224c802023-01-13 12:48:43 -0800372 int (*get_hba_mac)(struct ufs_hba *hba);
Asutosh Das2468da62023-01-13 12:48:45 -0800373 int (*op_runtime_config)(struct ufs_hba *hba);
Asutosh Dasf87b2c42023-01-13 12:48:50 -0800374 int (*get_outstanding_cqs)(struct ufs_hba *hba,
375 unsigned long *ocqs);
Can Guoedb0db02022-12-14 19:06:20 -0800376 int (*config_esi)(struct ufs_hba *hba);
Bart Van Assche858231b2023-09-21 12:22:47 -0700377 void (*config_scsi_dev)(struct scsi_device *sdev);
Sujit Reddy Thumma5c0c28a2014-09-25 15:32:21 +0300378};
379
Sahitya Tummala1ab27c92014-09-25 15:32:32 +0300380/* clock gating state */
381enum clk_gating_state {
382 CLKS_OFF,
383 CLKS_ON,
384 REQ_CLKS_OFF,
385 REQ_CLKS_ON,
386};
387
388/**
389 * struct ufs_clk_gating - UFS clock gating related info
390 * @gate_work: worker to turn off clocks after some delay as specified in
391 * delay_ms
392 * @ungate_work: worker to turn on clocks that will be used in case of
393 * interrupt context
394 * @state: the current clocks state
395 * @delay_ms: gating delay in ms
396 * @is_suspended: clk gating is suspended when set to 1 which can be used
397 * during suspend/resume
398 * @delay_attr: sysfs attribute to control delay_attr
Sahitya Tummalab4274112016-12-22 18:40:39 -0800399 * @enable_attr: sysfs attribute to enable/disable clock gating
400 * @is_enabled: Indicates the current status of clock gating
Can Guo4543d9d2021-01-20 02:04:22 -0800401 * @is_initialized: Indicates whether clock gating is initialized or not
Sahitya Tummala1ab27c92014-09-25 15:32:32 +0300402 * @active_reqs: number of requests that are pending and should be waited for
403 * completion before gating clocks.
Bart Van Asschecff91da2022-04-19 15:58:07 -0700404 * @clk_gating_workq: workqueue for clock gating work.
Sahitya Tummala1ab27c92014-09-25 15:32:32 +0300405 */
406struct ufs_clk_gating {
407 struct delayed_work gate_work;
408 struct work_struct ungate_work;
409 enum clk_gating_state state;
410 unsigned long delay_ms;
411 bool is_suspended;
412 struct device_attribute delay_attr;
Sahitya Tummalab4274112016-12-22 18:40:39 -0800413 struct device_attribute enable_attr;
414 bool is_enabled;
Can Guo4543d9d2021-01-20 02:04:22 -0800415 bool is_initialized;
Sahitya Tummala1ab27c92014-09-25 15:32:32 +0300416 int active_reqs;
Vijay Viswanath10e5e372018-05-03 16:37:22 +0530417 struct workqueue_struct *clk_gating_workq;
Sahitya Tummala1ab27c92014-09-25 15:32:32 +0300418};
419
subhashj@codeaurora.org401f1e42017-02-03 16:57:39 -0800420/**
421 * struct ufs_clk_scaling - UFS clock scaling related data
422 * @active_reqs: number of requests that are pending. If this is zero when
423 * devfreq ->target() function is called then schedule "suspend_work" to
424 * suspend devfreq.
425 * @tot_busy_t: Total busy time in current polling window
426 * @window_start_t: Start time (in jiffies) of the current polling window
427 * @busy_start_t: Start time of current busy period
428 * @enable_attr: sysfs attribute to enable/disable clock scaling
429 * @saved_pwr_info: UFS power mode may also be changed during scaling and this
430 * one keeps track of previous power mode.
431 * @workq: workqueue to schedule devfreq suspend/resume work
432 * @suspend_work: worker to suspend devfreq
433 * @resume_work: worker to resume devfreq
Manivannan Sadhasivam930bd772023-10-12 22:51:26 +0530434 * @target_freq: frequency requested by devfreq framework
Can Guo29b87e92020-11-26 17:58:48 -0800435 * @min_gear: lowest HS gear to scale down to
Can Guo0e9d4ca2021-01-20 02:04:21 -0800436 * @is_enabled: tracks if scaling is currently enabled or not, controlled by
Bart Van Asschecff91da2022-04-19 15:58:07 -0700437 * clkscale_enable sysfs node
Can Guo0e9d4ca2021-01-20 02:04:21 -0800438 * @is_allowed: tracks if scaling is currently allowed or not, used to block
Bart Van Asschecff91da2022-04-19 15:58:07 -0700439 * clock scaling which is not invoked from devfreq governor
Can Guo4543d9d2021-01-20 02:04:22 -0800440 * @is_initialized: Indicates whether clock scaling is initialized or not
subhashj@codeaurora.org401f1e42017-02-03 16:57:39 -0800441 * @is_busy_started: tracks if busy period has started or not
442 * @is_suspended: tracks if devfreq is suspended or not
443 */
Sahitya Tummala856b3482014-09-25 15:32:34 +0300444struct ufs_clk_scaling {
subhashj@codeaurora.org401f1e42017-02-03 16:57:39 -0800445 int active_reqs;
446 unsigned long tot_busy_t;
Stanley Chub1bf66d2020-06-11 18:10:43 +0800447 ktime_t window_start_t;
subhashj@codeaurora.org401f1e42017-02-03 16:57:39 -0800448 ktime_t busy_start_t;
Sahitya Tummalafcb0c4b2016-12-22 18:40:50 -0800449 struct device_attribute enable_attr;
Stanley Chu543a827b2023-03-30 09:29:18 +0800450 struct ufs_pa_layer_attr saved_pwr_info;
subhashj@codeaurora.org401f1e42017-02-03 16:57:39 -0800451 struct workqueue_struct *workq;
452 struct work_struct suspend_work;
453 struct work_struct resume_work;
Manivannan Sadhasivam930bd772023-10-12 22:51:26 +0530454 unsigned long target_freq;
Can Guo29b87e92020-11-26 17:58:48 -0800455 u32 min_gear;
Can Guo0e9d4ca2021-01-20 02:04:21 -0800456 bool is_enabled;
subhashj@codeaurora.org401f1e42017-02-03 16:57:39 -0800457 bool is_allowed;
Can Guo4543d9d2021-01-20 02:04:22 -0800458 bool is_initialized;
subhashj@codeaurora.org401f1e42017-02-03 16:57:39 -0800459 bool is_busy_started;
460 bool is_suspended;
Sahitya Tummala856b3482014-09-25 15:32:34 +0300461};
462
Stanley Chue965e5e2020-12-05 19:58:59 +0800463#define UFS_EVENT_HIST_LENGTH 8
Dolev Ravivff8e20c2016-12-22 18:42:18 -0800464/**
Stanley Chue965e5e2020-12-05 19:58:59 +0800465 * struct ufs_event_hist - keeps history of errors
Dolev Ravivff8e20c2016-12-22 18:42:18 -0800466 * @pos: index to indicate cyclic buffer position
Bart Van Asschecff91da2022-04-19 15:58:07 -0700467 * @val: cyclic buffer for registers value
Dolev Ravivff8e20c2016-12-22 18:42:18 -0800468 * @tstamp: cyclic buffer for time stamp
Adrian Hunterb6cacaf2021-01-07 09:25:38 +0200469 * @cnt: error counter
Dolev Ravivff8e20c2016-12-22 18:42:18 -0800470 */
Stanley Chue965e5e2020-12-05 19:58:59 +0800471struct ufs_event_hist {
Dolev Ravivff8e20c2016-12-22 18:42:18 -0800472 int pos;
Stanley Chue965e5e2020-12-05 19:58:59 +0800473 u32 val[UFS_EVENT_HIST_LENGTH];
Daniil Lunev0f85e742022-08-04 06:50:34 +1000474 u64 tstamp[UFS_EVENT_HIST_LENGTH];
Adrian Hunterb6cacaf2021-01-07 09:25:38 +0200475 unsigned long long cnt;
Dolev Ravivff8e20c2016-12-22 18:42:18 -0800476};
477
478/**
479 * struct ufs_stats - keeps usage/err statistics
Can Guo3f8af602020-08-09 05:15:50 -0700480 * @last_intr_status: record the last interrupt status.
481 * @last_intr_ts: record the last interrupt timestamp.
Dolev Ravivff8e20c2016-12-22 18:42:18 -0800482 * @hibern8_exit_cnt: Counter to keep track of number of exits,
483 * reset this after link-startup.
484 * @last_hibern8_exit_tstamp: Set time after the hibern8 exit.
485 * Clear after the first successful command completion.
Bart Van Asschecff91da2022-04-19 15:58:07 -0700486 * @event: array with event history.
Dolev Ravivff8e20c2016-12-22 18:42:18 -0800487 */
488struct ufs_stats {
Can Guo3f8af602020-08-09 05:15:50 -0700489 u32 last_intr_status;
Daniil Lunev0f85e742022-08-04 06:50:34 +1000490 u64 last_intr_ts;
Can Guo3f8af602020-08-09 05:15:50 -0700491
Dolev Ravivff8e20c2016-12-22 18:42:18 -0800492 u32 hibern8_exit_cnt;
Daniil Lunev0f85e742022-08-04 06:50:34 +1000493 u64 last_hibern8_exit_tstamp;
Stanley Chue965e5e2020-12-05 19:58:59 +0800494 struct ufs_event_hist event[UFS_EVT_CNT];
Dolev Ravivff8e20c2016-12-22 18:42:18 -0800495};
496
Bart Van Assche9c202092021-07-21 20:34:29 -0700497/**
498 * enum ufshcd_state - UFS host controller state
499 * @UFSHCD_STATE_RESET: Link is not operational. Postpone SCSI command
500 * processing.
501 * @UFSHCD_STATE_OPERATIONAL: The host controller is operational and can process
502 * SCSI commands.
503 * @UFSHCD_STATE_EH_SCHEDULED_NON_FATAL: The error handler has been scheduled.
504 * SCSI commands may be submitted to the controller.
505 * @UFSHCD_STATE_EH_SCHEDULED_FATAL: The error handler has been scheduled. Fail
506 * newly submitted SCSI commands with error code DID_BAD_TARGET.
507 * @UFSHCD_STATE_ERROR: An unrecoverable error occurred, e.g. link recovery
508 * failed. Fail all SCSI commands with error code DID_ERROR.
509 */
510enum ufshcd_state {
511 UFSHCD_STATE_RESET,
512 UFSHCD_STATE_OPERATIONAL,
513 UFSHCD_STATE_EH_SCHEDULED_NON_FATAL,
514 UFSHCD_STATE_EH_SCHEDULED_FATAL,
515 UFSHCD_STATE_ERROR,
516};
517
Christoph Hellwigc3f7d1f2020-02-21 06:08:12 -0800518enum ufshcd_quirks {
519 /* Interrupt aggregation support is broken */
520 UFSHCD_QUIRK_BROKEN_INTR_AGGR = 1 << 0,
521
522 /*
523 * delay before each dme command is required as the unipro
524 * layer has shown instabilities
525 */
526 UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS = 1 << 1,
527
528 /*
529 * If UFS host controller is having issue in processing LCC (Line
530 * Control Command) coming from device then enable this quirk.
531 * When this quirk is enabled, host controller driver should disable
532 * the LCC transmission on UFS device (by clearing TX_LCC_ENABLE
533 * attribute of device to 0).
534 */
535 UFSHCD_QUIRK_BROKEN_LCC = 1 << 2,
536
537 /*
538 * The attribute PA_RXHSUNTERMCAP specifies whether or not the
539 * inbound Link supports unterminated line in HS mode. Setting this
540 * attribute to 1 fixes moving to HS gear.
541 */
542 UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP = 1 << 3,
543
544 /*
545 * This quirk needs to be enabled if the host controller only allows
546 * accessing the peer dme attributes in AUTO mode (FAST AUTO or
547 * SLOW AUTO).
548 */
549 UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE = 1 << 4,
550
551 /*
552 * This quirk needs to be enabled if the host controller doesn't
553 * advertise the correct version in UFS_VER register. If this quirk
554 * is enabled, standard UFS host driver will call the vendor specific
555 * ops (get_ufs_hci_version) to get the correct version.
556 */
557 UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION = 1 << 5,
Alim Akhtar87183842020-05-28 06:46:49 +0530558
559 /*
560 * Clear handling for transfer/task request list is just opposite.
561 */
562 UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR = 1 << 6,
Alim Akhtarb638b5e2020-05-28 06:46:50 +0530563
564 /*
565 * This quirk needs to be enabled if host controller doesn't allow
566 * that the interrupt aggregation timer and counter are reset by s/w.
567 */
568 UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR = 1 << 7,
Alim Akhtar39bf2d82020-05-28 06:46:51 +0530569
570 /*
571 * This quirks needs to be enabled if host controller cannot be
572 * enabled via HCE register.
573 */
574 UFSHCI_QUIRK_BROKEN_HCE = 1 << 8,
Alim Akhtar26f968d2020-05-28 06:46:52 +0530575
576 /*
577 * This quirk needs to be enabled if the host controller regards
578 * resolution of the values of PRDTO and PRDTL in UTRD as byte.
579 */
580 UFSHCD_QUIRK_PRDT_BYTE_GRAN = 1 << 9,
Kiwoong Kimd779a6e2020-05-28 06:46:53 +0530581
582 /*
583 * This quirk needs to be enabled if the host controller reports
584 * OCS FATAL ERROR with device error through sense data
585 */
586 UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR = 1 << 10,
Kiwoong Kim5df6f2d2020-08-25 10:43:15 +0900587
588 /*
Adrian Hunter8da76f72020-08-10 17:10:24 +0300589 * This quirk needs to be enabled if the host controller has
590 * auto-hibernate capability but it doesn't work.
591 */
592 UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8 = 1 << 11,
Martin K. Petersen02f74152020-09-15 11:24:32 -0400593
594 /*
Kiwoong Kim5df6f2d2020-08-25 10:43:15 +0900595 * This quirk needs to disable manual flush for write booster
596 */
Martin K. Petersen02f74152020-09-15 11:24:32 -0400597 UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL = 1 << 12,
598
Kiwoong Kimb1d0d2e2020-12-21 10:24:40 +0900599 /*
600 * This quirk needs to disable unipro timeout values
601 * before power mode change
602 */
603 UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING = 1 << 13,
604
Kiwoong Kim2b2bfc8a2021-01-19 12:33:41 +0900605 /*
jongmin jeonga22bcfd2021-10-18 21:42:02 +0900606 * This quirk needs to be enabled if the host controller does not
607 * support UIC command
608 */
609 UFSHCD_QUIRK_BROKEN_UIC_CMD = 1 << 15,
jongmin jeong10fb4f82021-10-18 21:42:03 +0900610
611 /*
612 * This quirk needs to be enabled if the host controller cannot
613 * support physical host configuration.
614 */
615 UFSHCD_QUIRK_SKIP_PH_CONFIGURATION = 1 << 16,
Yoshihiro Shimoda65544002022-06-03 20:05:19 +0900616
617 /*
618 * This quirk needs to be enabled if the host controller has
619 * 64-bit addressing supported capability but it doesn't work.
620 */
621 UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS = 1 << 17,
Yoshihiro Shimoda2f11bbc2022-06-03 20:05:20 +0900622
623 /*
624 * This quirk needs to be enabled if the host controller has
625 * auto-hibernate capability but it's FASTAUTO only.
626 */
627 UFSHCD_QUIRK_HIBERN_FASTAUTO = 1 << 18,
Manivannan Sadhasivam96a71412022-12-22 19:39:57 +0530628
629 /*
630 * This quirk needs to be enabled if the host controller needs
631 * to reinit the device after switching to maximum gear.
632 */
633 UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH = 1 << 19,
Po-Wen Kaoc4ad4f22023-06-12 16:58:09 +0800634
635 /*
636 * Some host raises interrupt (per queue) in addition to
637 * CQES (traditional) when ESI is disabled.
638 * Enable this quirk will disable CQES and use per queue interrupt.
639 */
640 UFSHCD_QUIRK_MCQ_BROKEN_INTR = 1 << 20,
Po-Wen Kaoaa9d5d02023-06-12 16:58:10 +0800641
642 /*
643 * Some host does not implement SQ Run Time Command (SQRTC) register
644 * thus need this quirk to skip related flow.
645 */
646 UFSHCD_QUIRK_MCQ_BROKEN_RTC = 1 << 21,
Christoph Hellwigc3f7d1f2020-02-21 06:08:12 -0800647};
648
Stanley Chuc2014682020-03-18 18:40:11 +0800649enum ufshcd_caps {
650 /* Allow dynamic clk gating */
651 UFSHCD_CAP_CLK_GATING = 1 << 0,
652
653 /* Allow hiberb8 with clk gating */
654 UFSHCD_CAP_HIBERN8_WITH_CLK_GATING = 1 << 1,
655
656 /* Allow dynamic clk scaling */
657 UFSHCD_CAP_CLK_SCALING = 1 << 2,
658
659 /* Allow auto bkops to enabled during runtime suspend */
660 UFSHCD_CAP_AUTO_BKOPS_SUSPEND = 1 << 3,
661
662 /*
663 * This capability allows host controller driver to use the UFS HCI's
664 * interrupt aggregation capability.
665 * CAUTION: Enabling this might reduce overall UFS throughput.
666 */
667 UFSHCD_CAP_INTR_AGGR = 1 << 4,
668
669 /*
670 * This capability allows the device auto-bkops to be always enabled
671 * except during suspend (both runtime and suspend).
672 * Enabling this capability means that device will always be allowed
673 * to do background operation when it's active but it might degrade
674 * the performance of ongoing read/write operations.
675 */
676 UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND = 1 << 5,
677
678 /*
679 * This capability allows host controller driver to automatically
680 * enable runtime power management by itself instead of waiting
681 * for userspace to control the power management.
682 */
683 UFSHCD_CAP_RPM_AUTOSUSPEND = 1 << 6,
Asutosh Das3d17b9b2020-04-22 14:41:42 -0700684
685 /*
686 * This capability allows the host controller driver to turn-on
687 * WriteBooster, if the underlying device supports it and is
688 * provisioned to be used. This would increase the write performance.
689 */
690 UFSHCD_CAP_WB_EN = 1 << 7,
Satya Tangirala5e7341e2020-07-06 20:04:12 +0000691
692 /*
693 * This capability allows the host controller driver to use the
694 * inline crypto engine, if it is present
695 */
696 UFSHCD_CAP_CRYPTO = 1 << 8,
Can Guodd7143e2020-10-27 12:10:36 -0700697
698 /*
699 * This capability allows the controller regulators to be put into
700 * lpm mode aggressively during clock gating.
701 * This would increase power savings.
702 */
703 UFSHCD_CAP_AGGR_POWER_COLLAPSE = 1 << 9,
Adrian Hunterfe1d4c22020-11-03 16:14:02 +0200704
705 /*
706 * This capability allows the host controller driver to use DeepSleep,
707 * if it is supported by the UFS device. The host controller driver must
708 * support device hardware reset via the hba->device_reset() callback,
709 * in order to exit DeepSleep state.
710 */
711 UFSHCD_CAP_DEEPSLEEP = 1 << 10,
Avri Altmane88e2d32021-09-15 09:04:06 +0300712
713 /*
714 * This capability allows the host controller driver to use temperature
715 * notification if it is supported by the UFS device.
716 */
717 UFSHCD_CAP_TEMP_NOTIF = 1 << 11,
Peter Wang87bd0502022-08-04 10:54:22 +0800718
719 /*
720 * Enable WriteBooster when scaling up the clock and disable
721 * WriteBooster when scaling the clock down.
722 */
723 UFSHCD_CAP_WB_WITH_CLK_SCALING = 1 << 12,
Stanley Chuc2014682020-03-18 18:40:11 +0800724};
725
Stanley Chu90b84912020-05-09 17:37:13 +0800726struct ufs_hba_variant_params {
727 struct devfreq_dev_profile devfreq_profile;
728 struct devfreq_simple_ondemand_data ondemand_data;
729 u16 hba_enable_delay_us;
Stanley Chud14734ae2020-05-09 17:37:15 +0800730 u32 wb_flush_threshold;
Stanley Chu90b84912020-05-09 17:37:13 +0800731};
732
Can Guo1d8613a2021-04-21 19:28:39 -0700733struct ufs_hba_monitor {
734 unsigned long chunk_size;
735
736 unsigned long nr_sec_rw[2];
737 ktime_t total_busy[2];
738
739 unsigned long nr_req[2];
740 /* latencies*/
741 ktime_t lat_sum[2];
742 ktime_t lat_max[2];
743 ktime_t lat_min[2];
744
745 u32 nr_queued[2];
746 ktime_t busy_start_ts[2];
747
748 ktime_t enabled_ts;
749 bool enabled;
750};
751
Yaniv Gardi3a4bf062014-09-25 15:32:27 +0300752/**
Asutosh Dasc263b4e2023-01-13 12:48:42 -0800753 * struct ufshcd_res_info_t - MCQ related resource regions
754 *
755 * @name: resource name
756 * @resource: pointer to resource region
757 * @base: register base address
758 */
759struct ufshcd_res_info {
760 const char *name;
761 struct resource *resource;
762 void __iomem *base;
763};
764
765enum ufshcd_res {
766 RES_UFS,
767 RES_MCQ,
768 RES_MCQ_SQD,
769 RES_MCQ_SQIS,
770 RES_MCQ_CQD,
771 RES_MCQ_CQIS,
772 RES_MCQ_VS,
773 RES_MAX,
774};
775
776/**
Asutosh Das2468da62023-01-13 12:48:45 -0800777 * struct ufshcd_mcq_opr_info_t - Operation and Runtime registers
778 *
779 * @offset: Doorbell Address Offset
780 * @stride: Steps proportional to queue [0...31]
781 * @base: base address
782 */
783struct ufshcd_mcq_opr_info_t {
784 unsigned long offset;
785 unsigned long stride;
786 void __iomem *base;
787};
788
789enum ufshcd_mcq_opr {
790 OPR_SQD,
791 OPR_SQIS,
792 OPR_CQD,
793 OPR_CQIS,
794 OPR_MAX,
795};
796
797/**
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530798 * struct ufs_hba - per adapter private structure
799 * @mmio_base: UFSHCI base register address
800 * @ucdl_base_addr: UFS Command Descriptor base address
801 * @utrdl_base_addr: UTP Transfer Request Descriptor base address
802 * @utmrdl_base_addr: UTP Task Management Descriptor base address
803 * @ucdl_dma_addr: UFS Command Descriptor DMA address
804 * @utrdl_dma_addr: UTRDL DMA address
805 * @utmrdl_dma_addr: UTMRDL DMA address
806 * @host: Scsi_Host instance of the driver
807 * @dev: device handle
Bart Van Asschee2106582022-04-19 15:57:59 -0700808 * @ufs_device_wlun: WLUN that controls the entire UFS device.
Bart Van Asschecff91da2022-04-19 15:58:07 -0700809 * @hwmon_device: device instance registered with the hwmon core.
810 * @curr_dev_pwr_mode: active UFS device power mode.
811 * @uic_link_state: active state of the link to the UFS device.
812 * @rpm_lvl: desired UFS power management level during runtime PM.
813 * @spm_lvl: desired UFS power management level during system PM.
814 * @pm_op_in_progress: whether or not a PM operation is in progress.
815 * @ahit: value of Auto-Hibernate Idle Timer register.
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530816 * @lrb: local reference block
817 * @outstanding_tasks: Bits representing outstanding task requests
Bart Van Assche169f5eb2021-07-21 20:34:34 -0700818 * @outstanding_lock: Protects @outstanding_reqs.
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530819 * @outstanding_reqs: Bits representing outstanding transfer requests
820 * @capabilities: UFS Controller Capabilities
Asutosh Das6e1d8502023-01-13 12:48:38 -0800821 * @mcq_capabilities: UFS Multi Circular Queue capabilities
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530822 * @nutrs: Transfer Request Queue depth supported by controller
823 * @nutmrs: Task Management Queue depth supported by controller
Bart Van Assche945c3cc2021-12-03 15:19:42 -0800824 * @reserved_slot: Used to submit device commands. Protected by @dev_cmd.lock.
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530825 * @ufs_version: UFS Version to which controller complies
Sujit Reddy Thumma5c0c28a2014-09-25 15:32:21 +0300826 * @vops: pointer to variant specific operations
Bart Van Asschecff91da2022-04-19 15:58:07 -0700827 * @vps: pointer to variant specific parameters
Sujit Reddy Thumma5c0c28a2014-09-25 15:32:21 +0300828 * @priv: pointer to variant specific private data
Eric Biggersada1e652022-12-08 15:43:58 -0800829 * @sg_entry_size: size of struct ufshcd_sg_entry (may include variant fields)
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530830 * @irq: Irq number of the controller
Bart Van Asschecff91da2022-04-19 15:58:07 -0700831 * @is_irq_enabled: whether or not the UFS controller interrupt is enabled.
832 * @dev_ref_clk_freq: reference clock frequency
833 * @quirks: bitmask with information about deviations from the UFSHCI standard.
834 * @dev_quirks: bitmask with information about deviations from the UFS standard.
Bart Van Assche69a6c262019-12-09 10:13:09 -0800835 * @tmf_tag_set: TMF tag set.
836 * @tmf_queue: Used to allocate TMF tags.
Bart Van Asschecff91da2022-04-19 15:58:07 -0700837 * @tmf_rqs: array with pointers to TMF requests while these are in progress.
838 * @active_uic_cmd: handle of active UIC command
839 * @uic_cmd_mutex: mutex for UIC command
840 * @uic_async_done: completion used during UIC processing
Bart Van Assche9c202092021-07-21 20:34:29 -0700841 * @ufshcd_state: UFSHCD state
Sujit Reddy Thumma3441da72014-05-26 10:59:14 +0530842 * @eh_flags: Error handling flags
Seungwon Jeon2fbd0092013-06-26 22:39:27 +0530843 * @intr_mask: Interrupt Mask Bits
Sujit Reddy Thumma66ec6d52013-07-30 00:35:59 +0530844 * @ee_ctrl_mask: Exception event control mask
Bart Van Asschecff91da2022-04-19 15:58:07 -0700845 * @ee_drv_mask: Exception event mask for driver
846 * @ee_usr_mask: Exception event mask for user (set via debugfs)
847 * @ee_ctrl_mutex: Used to serialize exception event information.
Sujit Reddy Thumma1d337ec2014-09-25 15:32:26 +0300848 * @is_powered: flag to check if HBA is powered
Can Guo9cd20d32021-01-13 19:13:28 -0800849 * @shutting_down: flag to check if shutdown has been invoked
850 * @host_sem: semaphore used to serialize concurrent contexts
Adrian Hunter88b099002021-09-17 17:43:49 +0300851 * @eh_wq: Workqueue that eh_work works on
852 * @eh_work: Worker to handle UFS errors that require s/w attention
Sujit Reddy Thumma66ec6d52013-07-30 00:35:59 +0530853 * @eeh_work: Worker to handle exception events
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530854 * @errors: HBA errors
Sujit Reddy Thummae8e7f272014-05-26 10:59:15 +0530855 * @uic_error: UFS interconnect layer error status
856 * @saved_err: sticky error mask
857 * @saved_uic_err: sticky UIC error mask
Bart Van Asschecff91da2022-04-19 15:58:07 -0700858 * @ufs_stats: various error counters
Can Guo4db7a232020-08-09 05:15:51 -0700859 * @force_reset: flag to force eh_work perform a full reset
Can Guo2355b662020-08-24 19:07:06 -0700860 * @force_pmc: flag to force a power mode change
Can Guo2df74b62019-11-25 22:53:33 -0800861 * @silence_err_logs: flag to silence error logs
Sujit Reddy Thumma5a0b0cb2013-07-30 00:35:57 +0530862 * @dev_cmd: ufs device management command information
Yaniv Gardicad2e032015-03-31 17:37:14 +0300863 * @last_dme_cmd_tstamp: time stamp of the last completed DME command
Bart Van Asschecff91da2022-04-19 15:58:07 -0700864 * @nop_out_timeout: NOP OUT timeout value
865 * @dev_info: information about the UFS device
Sujit Reddy Thumma66ec6d52013-07-30 00:35:59 +0530866 * @auto_bkops_enabled: to track whether bkops is enabled in device
Sujit Reddy Thummaaa497612014-09-25 15:32:22 +0300867 * @vreg_info: UFS device voltage regulator information
Sujit Reddy Thummac6e79da2014-09-25 15:32:23 +0300868 * @clk_list_head: UFS host controller clocks list node head
Manivannan Sadhasivam930bd772023-10-12 22:51:26 +0530869 * @use_pm_opp: Indicates whether OPP based scaling is used or not
Bart Van Asschecff91da2022-04-19 15:58:07 -0700870 * @req_abort_count: number of times ufshcd_abort() has been called
871 * @lanes_per_direction: number of lanes per data direction between the UFS
872 * controller and the UFS device.
Dolev Raviv7eb584d2014-09-25 15:32:31 +0300873 * @pwr_info: holds current power mode
874 * @max_pwr_info: keeps the device max valid pwm
Bart Van Asschecff91da2022-04-19 15:58:07 -0700875 * @clk_gating: information related to clock gating
876 * @caps: bitmask with information about UFS controller capabilities
877 * @devfreq: frequency scaling information owned by the devfreq core
878 * @clk_scaling: frequency scaling information owned by the UFS driver
Bart Van Assche1a547cb2022-10-18 13:29:56 -0700879 * @system_suspending: system suspend has been started and system resume has
880 * not yet finished.
881 * @is_sys_suspended: UFS device has been suspended because of system suspend
Yaniv Gardiafdfff52016-03-10 17:37:15 +0200882 * @urgent_bkops_lvl: keeps track of urgent bkops level for device
883 * @is_urgent_bkops_lvl_checked: keeps track if the urgent bkops level for
884 * device is known or not.
Johan Hovoldba810432023-01-16 17:12:01 +0100885 * @wb_mutex: used to serialize devfreq and sysfs write booster toggling
Bart Van Asschecff91da2022-04-19 15:58:07 -0700886 * @clk_scaling_lock: used to serialize device commands and clock scaling
887 * @desc_size: descriptor sizes reported by device
Subhash Jadavani38135532018-05-03 16:37:18 +0530888 * @scsi_block_reqs_cnt: reference counting for scsi block requests
Bart Van Asschecff91da2022-04-19 15:58:07 -0700889 * @bsg_dev: struct device associated with the BSG queue
890 * @bsg_queue: BSG queue associated with the UFS controller
891 * @rpm_dev_flush_recheck_work: used to suspend from RPM (runtime power
892 * management) after the UFS device has finished a WriteBooster buffer
893 * flush or auto BKOP.
Bart Van Asschecff91da2022-04-19 15:58:07 -0700894 * @monitor: statistics about UFS commands
Satya Tangirala70297a82020-07-06 20:04:13 +0000895 * @crypto_capabilities: Content of crypto capabilities register (0x100)
896 * @crypto_cap_array: Array of crypto capabilities
897 * @crypto_cfg_register: Start of the crypto cfg array
Eric Biggerscb77cb52021-10-18 11:04:52 -0700898 * @crypto_profile: the crypto profile of this hba (if applicable)
Bart Van Asschecff91da2022-04-19 15:58:07 -0700899 * @debugfs_root: UFS controller debugfs root directory
900 * @debugfs_ee_work: used to restore ee_ctrl_mask after a delay
901 * @debugfs_ee_rate_limit_ms: user configurable delay after which to restore
902 * ee_ctrl_mask
903 * @luns_avail: number of regular and well known LUNs supported by the UFS
904 * device
Asutosh Das57b1c0e2023-01-13 12:48:41 -0800905 * @nr_hw_queues: number of hardware queues configured
906 * @nr_queues: number of Queues of different queue types
Bart Van Asschecff91da2022-04-19 15:58:07 -0700907 * @complete_put: whether or not to call ufshcd_rpm_put() from inside
908 * ufshcd_resume_complete()
Asutosh Das6e1d8502023-01-13 12:48:38 -0800909 * @ext_iid_sup: is EXT_IID is supported by UFSHC
Asutosh Das305a3572023-01-13 12:48:39 -0800910 * @mcq_sup: is mcq supported by UFSHC
Asutosh Das2468da62023-01-13 12:48:45 -0800911 * @mcq_enabled: is mcq ready to accept requests
Asutosh Dasc263b4e2023-01-13 12:48:42 -0800912 * @res: array of resource info of MCQ registers
913 * @mcq_base: Multi circular queue registers base address
Asutosh Das4682abf2023-01-13 12:48:44 -0800914 * @uhq: array of supported hardware queues
915 * @dev_cmd_queue: Queue for issuing device management commands
Bean Huo6bf999e2023-12-12 23:08:24 +0100916 * @mcq_opr: MCQ operation and runtime registers
917 * @ufs_rtc_update_work: A work for UFS RTC periodic update
Maramaina Naresh2777e732023-12-19 18:07:05 +0530918 * @pm_qos_req: PM QoS request handle
919 * @pm_qos_enabled: flag to check if pm qos is enabled
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530920 */
921struct ufs_hba {
922 void __iomem *mmio_base;
923
924 /* Virtual memory reference */
925 struct utp_transfer_cmd_desc *ucdl_base_addr;
926 struct utp_transfer_req_desc *utrdl_base_addr;
927 struct utp_task_req_desc *utmrdl_base_addr;
928
929 /* DMA memory reference */
930 dma_addr_t ucdl_dma_addr;
931 dma_addr_t utrdl_dma_addr;
932 dma_addr_t utmrdl_dma_addr;
933
934 struct Scsi_Host *host;
935 struct device *dev;
Bart Van Asschee2106582022-04-19 15:57:59 -0700936 struct scsi_device *ufs_device_wlun;
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530937
Avri Altmane88e2d32021-09-15 09:04:06 +0300938#ifdef CONFIG_SCSI_UFS_HWMON
939 struct device *hwmon_device;
940#endif
941
Subhash Jadavani57d104c2014-09-25 15:32:30 +0300942 enum ufs_dev_pwr_mode curr_dev_pwr_mode;
943 enum uic_link_state uic_link_state;
944 /* Desired UFS power management level during runtime PM */
945 enum ufs_pm_level rpm_lvl;
946 /* Desired UFS power management level during system PM */
947 enum ufs_pm_level spm_lvl;
948 int pm_op_in_progress;
949
Adrian Hunterad448372018-03-20 15:07:38 +0200950 /* Auto-Hibernate Idle Timer register value */
951 u32 ahit;
952
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530953 struct ufshcd_lrb *lrb;
954
955 unsigned long outstanding_tasks;
Bart Van Assche169f5eb2021-07-21 20:34:34 -0700956 spinlock_t outstanding_lock;
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530957 unsigned long outstanding_reqs;
958
959 u32 capabilities;
960 int nutrs;
Asutosh Das6e1d8502023-01-13 12:48:38 -0800961 u32 mcq_capabilities;
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530962 int nutmrs;
Bart Van Assche945c3cc2021-12-03 15:19:42 -0800963 u32 reserved_slot;
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530964 u32 ufs_version;
Arnd Bergmann176eb922019-03-04 20:39:11 +0100965 const struct ufs_hba_variant_ops *vops;
Stanley Chu90b84912020-05-09 17:37:13 +0800966 struct ufs_hba_variant_params *vps;
Sujit Reddy Thumma5c0c28a2014-09-25 15:32:21 +0300967 void *priv;
Eric Biggersada1e652022-12-08 15:43:58 -0800968#ifdef CONFIG_SCSI_UFS_VARIABLE_SG_ENTRY_SIZE
969 size_t sg_entry_size;
970#endif
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530971 unsigned int irq;
Subhash Jadavani57d104c2014-09-25 15:32:30 +0300972 bool is_irq_enabled;
Subhash Jadavani9e1e8a752018-10-16 14:29:41 +0530973 enum ufs_ref_clk_freq dev_ref_clk_freq;
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530974
Yaniv Gardicad2e032015-03-31 17:37:14 +0300975 unsigned int quirks; /* Deviations from standard UFSHCI spec. */
Seungwon Jeon6ccf44fe2013-06-26 22:39:29 +0530976
Yaniv Gardic58ab7a2016-03-10 17:37:10 +0200977 /* Device deviations from standard UFS device spec. */
978 unsigned int dev_quirks;
979
Bart Van Assche69a6c262019-12-09 10:13:09 -0800980 struct blk_mq_tag_set tmf_tag_set;
981 struct request_queue *tmf_queue;
Adrian Hunterf5ef3362021-09-22 12:10:59 +0300982 struct request **tmf_rqs;
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530983
Subhash Jadavani57d104c2014-09-25 15:32:30 +0300984 struct uic_command *active_uic_cmd;
985 struct mutex uic_cmd_mutex;
986 struct completion *uic_async_done;
Seungwon Jeon53b3d9c2013-08-31 21:40:22 +0530987
Bart Van Assche9c202092021-07-21 20:34:29 -0700988 enum ufshcd_state ufshcd_state;
Sujit Reddy Thumma3441da72014-05-26 10:59:14 +0530989 u32 eh_flags;
Seungwon Jeon2fbd0092013-06-26 22:39:27 +0530990 u32 intr_mask;
Bart Van Asschecff91da2022-04-19 15:58:07 -0700991 u16 ee_ctrl_mask;
992 u16 ee_drv_mask;
993 u16 ee_usr_mask;
Adrian Huntercd469472021-02-09 08:24:36 +0200994 struct mutex ee_ctrl_mutex;
Sujit Reddy Thumma1d337ec2014-09-25 15:32:26 +0300995 bool is_powered;
Can Guo9cd20d32021-01-13 19:13:28 -0800996 bool shutting_down;
997 struct semaphore host_sem;
Vinayak Holikattie0eca632013-02-25 21:44:33 +0530998
999 /* Work Queues */
Adrian Hunter88b099002021-09-17 17:43:49 +03001000 struct workqueue_struct *eh_wq;
1001 struct work_struct eh_work;
Sujit Reddy Thumma66ec6d52013-07-30 00:35:59 +05301002 struct work_struct eeh_work;
Vinayak Holikattie0eca632013-02-25 21:44:33 +05301003
1004 /* HBA Errors */
1005 u32 errors;
Sujit Reddy Thummae8e7f272014-05-26 10:59:15 +05301006 u32 uic_error;
1007 u32 saved_err;
1008 u32 saved_uic_err;
Dolev Ravivff8e20c2016-12-22 18:42:18 -08001009 struct ufs_stats ufs_stats;
Can Guo4db7a232020-08-09 05:15:51 -07001010 bool force_reset;
Can Guo2355b662020-08-24 19:07:06 -07001011 bool force_pmc;
Can Guo2df74b62019-11-25 22:53:33 -08001012 bool silence_err_logs;
Sujit Reddy Thumma5a0b0cb2013-07-30 00:35:57 +05301013
1014 /* Device management request data */
1015 struct ufs_dev_cmd dev_cmd;
Yaniv Gardicad2e032015-03-31 17:37:14 +03001016 ktime_t last_dme_cmd_tstamp;
Adrian Hunter1cbc9ad2021-08-31 17:53:17 +03001017 int nop_out_timeout;
Sujit Reddy Thumma66ec6d52013-07-30 00:35:59 +05301018
Subhash Jadavani57d104c2014-09-25 15:32:30 +03001019 /* Keeps information of the UFS device connected to this host */
1020 struct ufs_dev_info dev_info;
Sujit Reddy Thumma66ec6d52013-07-30 00:35:59 +05301021 bool auto_bkops_enabled;
Sujit Reddy Thummaaa497612014-09-25 15:32:22 +03001022 struct ufs_vreg_info vreg_info;
Sujit Reddy Thummac6e79da2014-09-25 15:32:23 +03001023 struct list_head clk_list_head;
Manivannan Sadhasivam930bd772023-10-12 22:51:26 +05301024 bool use_pm_opp;
Subhash Jadavani57d104c2014-09-25 15:32:30 +03001025
Gilad Broner7fabb772017-02-03 16:56:50 -08001026 /* Number of requests aborts */
1027 int req_abort_count;
1028
Yaniv Gardi54b879b2016-03-10 17:37:05 +02001029 /* Number of lanes available (1 or 2) for Rx/Tx */
1030 u32 lanes_per_direction;
Dolev Raviv7eb584d2014-09-25 15:32:31 +03001031 struct ufs_pa_layer_attr pwr_info;
1032 struct ufs_pwr_mode_info max_pwr_info;
Sahitya Tummala1ab27c92014-09-25 15:32:32 +03001033
1034 struct ufs_clk_gating clk_gating;
1035 /* Control to enable/disable host capabilities */
1036 u32 caps;
Sahitya Tummala856b3482014-09-25 15:32:34 +03001037
1038 struct devfreq *devfreq;
1039 struct ufs_clk_scaling clk_scaling;
Bart Van Assche1a547cb2022-10-18 13:29:56 -07001040 bool system_suspending;
Dolev Ravive7850602014-09-25 15:32:36 +03001041 bool is_sys_suspended;
Yaniv Gardiafdfff52016-03-10 17:37:15 +02001042
1043 enum bkops_status urgent_bkops_lvl;
1044 bool is_urgent_bkops_lvl_checked;
subhashj@codeaurora.orga3cd5ec2017-02-03 16:57:02 -08001045
Johan Hovoldba810432023-01-16 17:12:01 +01001046 struct mutex wb_mutex;
subhashj@codeaurora.orga3cd5ec2017-02-03 16:57:02 -08001047 struct rw_semaphore clk_scaling_lock;
Subhash Jadavani38135532018-05-03 16:37:18 +05301048 atomic_t scsi_block_reqs_cnt;
Avri Altmandf032bf2018-10-07 17:30:35 +03001049
1050 struct device bsg_dev;
1051 struct request_queue *bsg_queue;
Stanley Chu51dd9052020-05-22 16:32:12 +08001052 struct delayed_work rpm_dev_flush_recheck_work;
Satya Tangirala70297a82020-07-06 20:04:13 +00001053
Can Guo1d8613a2021-04-21 19:28:39 -07001054 struct ufs_hba_monitor monitor;
1055
Satya Tangirala70297a82020-07-06 20:04:13 +00001056#ifdef CONFIG_SCSI_UFS_CRYPTO
1057 union ufs_crypto_capabilities crypto_capabilities;
1058 union ufs_crypto_cap_entry *crypto_cap_array;
1059 u32 crypto_cfg_register;
Eric Biggerscb77cb52021-10-18 11:04:52 -07001060 struct blk_crypto_profile crypto_profile;
Satya Tangirala70297a82020-07-06 20:04:13 +00001061#endif
Adrian Hunterb6cacaf2021-01-07 09:25:38 +02001062#ifdef CONFIG_DEBUG_FS
1063 struct dentry *debugfs_root;
Adrian Hunter7deedfd2021-02-09 08:24:37 +02001064 struct delayed_work debugfs_ee_work;
1065 u32 debugfs_ee_rate_limit_ms;
Adrian Hunterb6cacaf2021-01-07 09:25:38 +02001066#endif
Akinobu Mita045da302023-11-18 21:44:43 +09001067#ifdef CONFIG_SCSI_UFS_FAULT_INJECTION
1068 struct fault_attr trigger_eh_attr;
1069 struct fault_attr timeout_attr;
1070#endif
Asutosh Dasb294ff32021-04-23 17:20:16 -07001071 u32 luns_avail;
Asutosh Das57b1c0e2023-01-13 12:48:41 -08001072 unsigned int nr_hw_queues;
1073 unsigned int nr_queues[HCTX_MAX_TYPES];
Asutosh Dasb294ff32021-04-23 17:20:16 -07001074 bool complete_put;
Asutosh Das6e1d8502023-01-13 12:48:38 -08001075 bool ext_iid_sup;
Asutosh Das0cab4022023-01-13 12:48:40 -08001076 bool scsi_host_added;
Asutosh Das305a3572023-01-13 12:48:39 -08001077 bool mcq_sup;
Asutosh Das2468da62023-01-13 12:48:45 -08001078 bool mcq_enabled;
Asutosh Dasc263b4e2023-01-13 12:48:42 -08001079 struct ufshcd_res_info res[RES_MAX];
1080 void __iomem *mcq_base;
Asutosh Das4682abf2023-01-13 12:48:44 -08001081 struct ufs_hw_queue *uhq;
1082 struct ufs_hw_queue *dev_cmd_queue;
Asutosh Das2468da62023-01-13 12:48:45 -08001083 struct ufshcd_mcq_opr_info_t mcq_opr[OPR_MAX];
Bean Huo6bf999e2023-12-12 23:08:24 +01001084
1085 struct delayed_work ufs_rtc_update_work;
Maramaina Naresh2777e732023-12-19 18:07:05 +05301086 struct pm_qos_request pm_qos_req;
1087 bool pm_qos_enabled;
Vinayak Holikattie0eca632013-02-25 21:44:33 +05301088};
1089
Asutosh Das4682abf2023-01-13 12:48:44 -08001090/**
1091 * struct ufs_hw_queue - per hardware queue structure
Asutosh Das2468da62023-01-13 12:48:45 -08001092 * @mcq_sq_head: base address of submission queue head pointer
1093 * @mcq_sq_tail: base address of submission queue tail pointer
1094 * @mcq_cq_head: base address of completion queue head pointer
1095 * @mcq_cq_tail: base address of completion queue tail pointer
Asutosh Das4682abf2023-01-13 12:48:44 -08001096 * @sqe_base_addr: submission queue entry base address
1097 * @sqe_dma_addr: submission queue dma address
1098 * @cqe_base_addr: completion queue base address
1099 * @cqe_dma_addr: completion queue dma address
1100 * @max_entries: max number of slots in this hardware queue
Asutosh Das2468da62023-01-13 12:48:45 -08001101 * @id: hardware queue ID
Asutosh Das22a2d562023-01-13 12:48:47 -08001102 * @sq_tp_slot: current slot to which SQ tail pointer is pointing
1103 * @sq_lock: serialize submission queue access
Asutosh Dasf87b2c42023-01-13 12:48:50 -08001104 * @cq_tail_slot: current slot to which CQ tail pointer is pointing
1105 * @cq_head_slot: current slot to which CQ head pointer is pointing
Asutosh Dased975062023-01-13 12:48:51 -08001106 * @cq_lock: Synchronize between multiple polling instances
Bao D. Nguyen8d729032023-05-29 15:12:22 -07001107 * @sq_mutex: prevent submission queue concurrent access
Asutosh Das4682abf2023-01-13 12:48:44 -08001108 */
1109struct ufs_hw_queue {
Asutosh Das2468da62023-01-13 12:48:45 -08001110 void __iomem *mcq_sq_head;
1111 void __iomem *mcq_sq_tail;
1112 void __iomem *mcq_cq_head;
1113 void __iomem *mcq_cq_tail;
1114
Avri Altman3c85f082023-03-29 13:13:03 +03001115 struct utp_transfer_req_desc *sqe_base_addr;
Asutosh Das4682abf2023-01-13 12:48:44 -08001116 dma_addr_t sqe_dma_addr;
1117 struct cq_entry *cqe_base_addr;
1118 dma_addr_t cqe_dma_addr;
1119 u32 max_entries;
Asutosh Das2468da62023-01-13 12:48:45 -08001120 u32 id;
Asutosh Das22a2d562023-01-13 12:48:47 -08001121 u32 sq_tail_slot;
1122 spinlock_t sq_lock;
Asutosh Dasf87b2c42023-01-13 12:48:50 -08001123 u32 cq_tail_slot;
1124 u32 cq_head_slot;
Asutosh Dased975062023-01-13 12:48:51 -08001125 spinlock_t cq_lock;
Bao D. Nguyen8d729032023-05-29 15:12:22 -07001126 /* prevent concurrent access to submission queue */
1127 struct mutex sq_mutex;
Vinayak Holikattie0eca632013-02-25 21:44:33 +05301128};
1129
Asutosh Das2468da62023-01-13 12:48:45 -08001130static inline bool is_mcq_enabled(struct ufs_hba *hba)
1131{
1132 return hba->mcq_enabled;
1133}
1134
Eric Biggersada1e652022-12-08 15:43:58 -08001135#ifdef CONFIG_SCSI_UFS_VARIABLE_SG_ENTRY_SIZE
1136static inline size_t ufshcd_sg_entry_size(const struct ufs_hba *hba)
1137{
1138 return hba->sg_entry_size;
1139}
1140
1141static inline void ufshcd_set_sg_entry_size(struct ufs_hba *hba, size_t sg_entry_size)
1142{
1143 WARN_ON_ONCE(sg_entry_size < sizeof(struct ufshcd_sg_entry));
1144 hba->sg_entry_size = sg_entry_size;
1145}
1146#else
1147static inline size_t ufshcd_sg_entry_size(const struct ufs_hba *hba)
1148{
1149 return sizeof(struct ufshcd_sg_entry);
1150}
1151
1152#define ufshcd_set_sg_entry_size(hba, sg_entry_size) \
1153 ({ (void)(hba); BUILD_BUG_ON(sg_entry_size != sizeof(struct ufshcd_sg_entry)); })
1154#endif
1155
Po-Wen Kao06caeb52023-05-04 23:44:50 +08001156static inline size_t ufshcd_get_ucd_size(const struct ufs_hba *hba)
Eric Biggersada1e652022-12-08 15:43:58 -08001157{
1158 return sizeof(struct utp_transfer_cmd_desc) + SG_ALL * ufshcd_sg_entry_size(hba);
1159}
1160
Sahitya Tummala1ab27c92014-09-25 15:32:32 +03001161/* Returns true if clocks can be gated. Otherwise false */
1162static inline bool ufshcd_is_clkgating_allowed(struct ufs_hba *hba)
1163{
1164 return hba->caps & UFSHCD_CAP_CLK_GATING;
1165}
1166static inline bool ufshcd_can_hibern8_during_gating(struct ufs_hba *hba)
1167{
1168 return hba->caps & UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;
1169}
Sahitya Tummalafcb0c4b2016-12-22 18:40:50 -08001170static inline int ufshcd_is_clkscaling_supported(struct ufs_hba *hba)
Sahitya Tummala856b3482014-09-25 15:32:34 +03001171{
1172 return hba->caps & UFSHCD_CAP_CLK_SCALING;
1173}
Subhash Jadavani374a2462014-09-25 15:32:35 +03001174static inline bool ufshcd_can_autobkops_during_suspend(struct ufs_hba *hba)
1175{
1176 return hba->caps & UFSHCD_CAP_AUTO_BKOPS_SUSPEND;
1177}
Stanley Chu49615ba2019-09-16 23:56:50 +08001178static inline bool ufshcd_is_rpm_autosuspend_allowed(struct ufs_hba *hba)
1179{
1180 return hba->caps & UFSHCD_CAP_RPM_AUTOSUSPEND;
1181}
Subhash Jadavani374a2462014-09-25 15:32:35 +03001182
Yaniv Gardib8521902015-05-17 18:54:57 +03001183static inline bool ufshcd_is_intr_aggr_allowed(struct ufs_hba *hba)
1184{
Keoseong Park1c0810e2021-06-28 14:58:01 +09001185 return (hba->caps & UFSHCD_CAP_INTR_AGGR) &&
1186 !(hba->quirks & UFSHCD_QUIRK_BROKEN_INTR_AGGR);
Yaniv Gardib8521902015-05-17 18:54:57 +03001187}
1188
Can Guodd7143e2020-10-27 12:10:36 -07001189static inline bool ufshcd_can_aggressive_pc(struct ufs_hba *hba)
1190{
1191 return !!(ufshcd_is_link_hibern8(hba) &&
1192 (hba->caps & UFSHCD_CAP_AGGR_POWER_COLLAPSE));
1193}
1194
Stanley Chuee5f1042019-05-21 14:44:52 +08001195static inline bool ufshcd_is_auto_hibern8_supported(struct ufs_hba *hba)
1196{
Adrian Hunter8da76f72020-08-10 17:10:24 +03001197 return (hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT) &&
1198 !(hba->quirks & UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8);
Stanley Chuee5f1042019-05-21 14:44:52 +08001199}
1200
Stanley Chu5a244e02020-01-29 18:52:50 +08001201static inline bool ufshcd_is_auto_hibern8_enabled(struct ufs_hba *hba)
1202{
Bart Van Assche51d16282022-04-19 15:57:46 -07001203 return FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK, hba->ahit);
Stanley Chu5a244e02020-01-29 18:52:50 +08001204}
1205
Asutosh Das3d17b9b2020-04-22 14:41:42 -07001206static inline bool ufshcd_is_wb_allowed(struct ufs_hba *hba)
1207{
1208 return hba->caps & UFSHCD_CAP_WB_EN;
1209}
1210
Peter Wang87bd0502022-08-04 10:54:22 +08001211static inline bool ufshcd_enable_wb_if_scaling_up(struct ufs_hba *hba)
1212{
1213 return hba->caps & UFSHCD_CAP_WB_WITH_CLK_SCALING;
1214}
1215
Asutosh Das2468da62023-01-13 12:48:45 -08001216#define ufsmcq_writel(hba, val, reg) \
1217 writel((val), (hba)->mcq_base + (reg))
1218#define ufsmcq_readl(hba, reg) \
1219 readl((hba)->mcq_base + (reg))
1220
1221#define ufsmcq_writelx(hba, val, reg) \
1222 writel_relaxed((val), (hba)->mcq_base + (reg))
1223#define ufsmcq_readlx(hba, reg) \
1224 readl_relaxed((hba)->mcq_base + (reg))
1225
Seungwon Jeonb873a2752013-06-26 22:39:26 +05301226#define ufshcd_writel(hba, val, reg) \
1227 writel((val), (hba)->mmio_base + (reg))
1228#define ufshcd_readl(hba, reg) \
1229 readl((hba)->mmio_base + (reg))
1230
Dolev Ravive7850602014-09-25 15:32:36 +03001231/**
Bart Van Asschecff91da2022-04-19 15:58:07 -07001232 * ufshcd_rmwl - perform read/modify/write for a controller register
1233 * @hba: per adapter instance
1234 * @mask: mask to apply on read value
1235 * @val: actual value to write
1236 * @reg: register address
Dolev Ravive7850602014-09-25 15:32:36 +03001237 */
1238static inline void ufshcd_rmwl(struct ufs_hba *hba, u32 mask, u32 val, u32 reg)
1239{
1240 u32 tmp;
1241
1242 tmp = ufshcd_readl(hba, reg);
1243 tmp &= ~mask;
1244 tmp |= (val & mask);
1245 ufshcd_writel(hba, tmp, reg);
1246}
1247
Manivannan Sadhasivam0ae7a022023-12-08 12:28:51 +05301248void ufshcd_enable_irq(struct ufs_hba *hba);
1249void ufshcd_disable_irq(struct ufs_hba *hba);
Sujit Reddy Thumma5c0c28a2014-09-25 15:32:21 +03001250int ufshcd_alloc_host(struct device *, struct ufs_hba **);
Yaniv Gardi47555a52015-10-28 13:15:49 +02001251void ufshcd_dealloc_host(struct ufs_hba *);
Stanley Chu9d19bf7a2020-01-17 11:51:07 +08001252int ufshcd_hba_enable(struct ufs_hba *hba);
Keoseong Parkecd7beb2021-05-18 21:12:17 +09001253int ufshcd_init(struct ufs_hba *, void __iomem *, unsigned int);
Stanley Chu087c5ef2020-03-27 17:53:28 +08001254int ufshcd_link_recovery(struct ufs_hba *hba);
Stanley Chu9d19bf7a2020-01-17 11:51:07 +08001255int ufshcd_make_hba_operational(struct ufs_hba *hba);
Vinayak Holikattie0eca632013-02-25 21:44:33 +05301256void ufshcd_remove(struct ufs_hba *);
Asutosh Das525943a2021-09-28 02:06:12 -07001257int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
Stanley Chu9d19bf7a2020-01-17 11:51:07 +08001258int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
Stanley Chu5c955c12020-03-18 18:40:12 +08001259void ufshcd_delay_us(unsigned long us, unsigned long tolerance);
Subhash Jadavani9e1e8a752018-10-16 14:29:41 +05301260void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk);
Stanley Chue965e5e2020-12-05 19:58:59 +08001261void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val);
Alice.Chao3a95f5b2021-05-28 11:36:21 +08001262void ufshcd_hba_stop(struct ufs_hba *hba);
Bart Van Assche267a59f2021-10-20 14:40:19 -07001263void ufshcd_schedule_eh_work(struct ufs_hba *hba);
Po-Wen Kao11afb652023-07-01 20:44:40 +08001264void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds);
1265u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i);
Can Guoe02288e2022-12-14 19:06:21 -08001266void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i);
Bao D. Nguyen57d6ef42023-05-29 15:12:25 -07001267unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba,
Can Guoe02288e2022-12-14 19:06:21 -08001268 struct ufs_hw_queue *hwq);
Po-Wen Kao11afb652023-07-01 20:44:40 +08001269void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba);
Can Guoe02288e2022-12-14 19:06:21 -08001270void ufshcd_mcq_enable_esi(struct ufs_hba *hba);
ChanWoo Leeab3e6c42024-01-02 10:42:22 +09001271void ufshcd_mcq_enable(struct ufs_hba *hba);
Can Guoe02288e2022-12-14 19:06:21 -08001272void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg);
Vinayak Holikattie0eca632013-02-25 21:44:33 +05301273
Manivannan Sadhasivam72208eb2023-10-12 22:51:27 +05301274int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table,
1275 struct dev_pm_opp *opp, void *data,
1276 bool scaling_down);
Yaniv Gardi1ce58982015-10-28 13:15:47 +02001277/**
1278 * ufshcd_set_variant - set variant specific data to the hba
Bart Van Asschecff91da2022-04-19 15:58:07 -07001279 * @hba: per adapter instance
1280 * @variant: pointer to variant specific data
Yaniv Gardi1ce58982015-10-28 13:15:47 +02001281 */
1282static inline void ufshcd_set_variant(struct ufs_hba *hba, void *variant)
1283{
1284 BUG_ON(!hba);
1285 hba->priv = variant;
1286}
1287
1288/**
1289 * ufshcd_get_variant - get variant specific data from the hba
Bart Van Asschecff91da2022-04-19 15:58:07 -07001290 * @hba: per adapter instance
Yaniv Gardi1ce58982015-10-28 13:15:47 +02001291 */
1292static inline void *ufshcd_get_variant(struct ufs_hba *hba)
1293{
1294 BUG_ON(!hba);
1295 return hba->priv;
1296}
Avri Altmane88e2d32021-09-15 09:04:06 +03001297
Bart Van Assche9bb25e52021-07-21 20:34:24 -07001298#ifdef CONFIG_PM
Bart Van Asschef1ecbe12021-07-21 20:34:23 -07001299extern int ufshcd_runtime_suspend(struct device *dev);
1300extern int ufshcd_runtime_resume(struct device *dev);
Bart Van Assche9bb25e52021-07-21 20:34:24 -07001301#endif
1302#ifdef CONFIG_PM_SLEEP
Bart Van Asschef1ecbe12021-07-21 20:34:23 -07001303extern int ufshcd_system_suspend(struct device *dev);
1304extern int ufshcd_system_resume(struct device *dev);
Anjana Hari88441a82023-02-02 21:40:45 +05301305extern int ufshcd_system_freeze(struct device *dev);
1306extern int ufshcd_system_thaw(struct device *dev);
1307extern int ufshcd_system_restore(struct device *dev);
Bart Van Assche9bb25e52021-07-21 20:34:24 -07001308#endif
Anjana Hari88441a82023-02-02 21:40:45 +05301309
Stanley Chufc85a74e2020-11-16 14:50:52 +08001310extern int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
1311 int agreed_gear,
1312 int adapt_val);
Seungwon Jeon12b4fdb2013-08-31 21:40:21 +05301313extern int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
1314 u8 attr_set, u32 mib_val, u8 peer);
1315extern int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
1316 u32 *mib_val, u8 peer);
Alim Akhtar0d846e72018-05-06 15:44:18 +05301317extern int ufshcd_config_pwr_mode(struct ufs_hba *hba,
1318 struct ufs_pa_layer_attr *desired_pwr_mode);
Stanley Chufc53683b2022-06-16 13:37:15 +08001319extern int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode);
Seungwon Jeon12b4fdb2013-08-31 21:40:21 +05301320
1321/* UIC command interfaces for DME primitives */
1322#define DME_LOCAL 0
1323#define DME_PEER 1
1324#define ATTR_SET_NOR 0 /* NORMAL */
1325#define ATTR_SET_ST 1 /* STATIC */
1326
1327static inline int ufshcd_dme_set(struct ufs_hba *hba, u32 attr_sel,
1328 u32 mib_val)
1329{
1330 return ufshcd_dme_set_attr(hba, attr_sel, ATTR_SET_NOR,
1331 mib_val, DME_LOCAL);
1332}
1333
1334static inline int ufshcd_dme_st_set(struct ufs_hba *hba, u32 attr_sel,
1335 u32 mib_val)
1336{
1337 return ufshcd_dme_set_attr(hba, attr_sel, ATTR_SET_ST,
1338 mib_val, DME_LOCAL);
1339}
1340
1341static inline int ufshcd_dme_peer_set(struct ufs_hba *hba, u32 attr_sel,
1342 u32 mib_val)
1343{
1344 return ufshcd_dme_set_attr(hba, attr_sel, ATTR_SET_NOR,
1345 mib_val, DME_PEER);
1346}
1347
1348static inline int ufshcd_dme_peer_st_set(struct ufs_hba *hba, u32 attr_sel,
1349 u32 mib_val)
1350{
1351 return ufshcd_dme_set_attr(hba, attr_sel, ATTR_SET_ST,
1352 mib_val, DME_PEER);
1353}
1354
1355static inline int ufshcd_dme_get(struct ufs_hba *hba,
1356 u32 attr_sel, u32 *mib_val)
1357{
1358 return ufshcd_dme_get_attr(hba, attr_sel, mib_val, DME_LOCAL);
1359}
1360
1361static inline int ufshcd_dme_peer_get(struct ufs_hba *hba,
1362 u32 attr_sel, u32 *mib_val)
1363{
1364 return ufshcd_dme_get_attr(hba, attr_sel, mib_val, DME_PEER);
1365}
1366
Yaniv Gardif37aabc2016-03-10 17:37:20 +02001367static inline bool ufshcd_is_hs_mode(struct ufs_pa_layer_attr *pwr_info)
1368{
1369 return (pwr_info->pwr_rx == FAST_MODE ||
1370 pwr_info->pwr_rx == FASTAUTO_MODE) &&
1371 (pwr_info->pwr_tx == FAST_MODE ||
1372 pwr_info->pwr_tx == FASTAUTO_MODE);
1373}
1374
Stanley Chu984eaac2020-02-07 15:03:57 +08001375static inline int ufshcd_disable_host_tx_lcc(struct ufs_hba *hba)
1376{
1377 return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0);
1378}
1379
Stanley Chuba7af5e2019-12-30 13:32:28 +08001380void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit);
Bart Van Asscheaead21f2022-04-19 15:57:53 -07001381void ufshcd_fixup_dev_quirks(struct ufs_hba *hba,
1382 const struct ufs_dev_quirk *fixups);
Tomas Winkler4b828fe2019-07-30 08:55:17 +03001383#define SD_ASCII_STD true
1384#define SD_RAW false
1385int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
1386 u8 **buf, bool ascii);
Stanislav Nijnikov2238d312018-02-15 14:14:07 +02001387
Bart Van Assche078f4f42023-05-29 13:26:40 -07001388void ufshcd_hold(struct ufs_hba *hba);
Sahitya Tummala1ab27c92014-09-25 15:32:32 +03001389void ufshcd_release(struct ufs_hba *hba);
Potomski, MichalXa4b0e8a2017-02-23 09:05:30 +00001390
Bart Van Asschead8a6472022-04-19 15:58:04 -07001391void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value);
1392
Yaniv Gardi37113102016-03-10 17:37:16 +02001393u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba);
Yaniv Gardi0263bcd2015-10-28 13:15:48 +02001394
Stanley Chu1d6f9de2022-06-16 13:37:23 +08001395int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg);
1396
Avri Altmane77044c52018-10-07 17:30:39 +03001397int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd);
1398
Bean Huo6ff265fc2022-12-01 15:04:37 +01001399int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu,
1400 struct utp_upiu_req *rsp_upiu, struct ufs_ehs *ehs_req,
1401 struct ufs_ehs *ehs_rsp, int sg_cnt,
1402 struct scatterlist *sg_list, enum dma_data_direction dir);
Yue Hu3b5f3c02021-03-18 17:55:36 +08001403int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable);
Jinyoung Choi6c4148c2022-08-04 16:53:54 +09001404int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable);
Asutosh Dasb294ff32021-04-23 17:20:16 -07001405int ufshcd_suspend_prepare(struct device *dev);
Adrian Hunterddba1cf2021-10-27 16:06:14 +03001406int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm);
Asutosh Dasb294ff32021-04-23 17:20:16 -07001407void ufshcd_resume_complete(struct device *dev);
Nitin Rawat548fdf72023-07-26 19:11:39 +05301408bool ufshcd_is_hba_active(struct ufs_hba *hba);
Maramaina Naresh2777e732023-12-19 18:07:05 +05301409void ufshcd_pm_qos_init(struct ufs_hba *hba);
1410void ufshcd_pm_qos_exit(struct ufs_hba *hba);
Bean Huo8e834ca2021-01-19 17:38:42 +01001411
Yaniv Gardi0263bcd2015-10-28 13:15:48 +02001412/* Wrapper functions for safely calling variant operations */
Yaniv Gardi0263bcd2015-10-28 13:15:48 +02001413static inline int ufshcd_vops_init(struct ufs_hba *hba)
1414{
1415 if (hba->vops && hba->vops->init)
1416 return hba->vops->init(hba);
1417
1418 return 0;
1419}
1420
Stanley Chu92bcebe2020-12-05 20:00:39 +08001421static inline int ufshcd_vops_phy_initialization(struct ufs_hba *hba)
1422{
1423 if (hba->vops && hba->vops->phy_initialization)
1424 return hba->vops->phy_initialization(hba);
1425
1426 return 0;
1427}
1428
Krzysztof Kozlowski35d11ec2022-06-23 12:24:32 +02001429extern const struct ufs_pm_lvl_states ufs_pm_lvl_states[];
Stanislav Nijnikovcbb68132018-02-15 14:14:01 +02001430
Tomas Winklerba809172018-06-14 11:14:09 +03001431int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
1432 const char *prefix);
1433
Adrian Hunter7deedfd2021-02-09 08:24:37 +02001434int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask);
1435int ufshcd_write_ee_control(struct ufs_hba *hba);
Krzysztof Kozlowski35d11ec2022-06-23 12:24:32 +02001436int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask,
1437 const u16 *other_mask, u16 set, u16 clr);
Adrian Huntercd469472021-02-09 08:24:36 +02001438
Vinayak Holikattie0eca632013-02-25 21:44:33 +05301439#endif /* End of Header */