| // SPDX-License-Identifier: GPL-2.0 |
| /* |
| * Universal Flash Storage Host Performance Booster |
| * |
| * Copyright (C) 2017-2021 Samsung Electronics Co., Ltd. |
| * |
| * Authors: |
| * Yongmyung Lee <ymhungry.lee@samsung.com> |
| * Jinyoung Choi <j-young.choi@samsung.com> |
| */ |
| |
| #include <asm/unaligned.h> |
| #include <linux/delay.h> |
| #include <linux/device.h> |
| #include <linux/module.h> |
| #include <scsi/scsi_cmnd.h> |
| |
| #include "ufshcd-priv.h" |
| #include "ufshpb.h" |
| #include "../../scsi/sd.h" |
| |
| #define ACTIVATION_THRESHOLD 8 /* 8 IOs */ |
| #define READ_TO_MS 1000 |
| #define READ_TO_EXPIRIES 100 |
| #define POLLING_INTERVAL_MS 200 |
| #define THROTTLE_MAP_REQ_DEFAULT 1 |
| |
| /* memory management */ |
| static struct kmem_cache *ufshpb_mctx_cache; |
| static mempool_t *ufshpb_mctx_pool; |
| static mempool_t *ufshpb_page_pool; |
| /* A cache size of 2MB can cache ppn in the 1GB range. */ |
| static unsigned int ufshpb_host_map_kbytes = 2048; |
| static int tot_active_srgn_pages; |
| |
| static struct workqueue_struct *ufshpb_wq; |
| |
| static void ufshpb_update_active_info(struct ufshpb_lu *hpb, int rgn_idx, |
| int srgn_idx); |
| |
| bool ufshpb_is_allowed(struct ufs_hba *hba) |
| { |
| return !(hba->ufshpb_dev.hpb_disabled); |
| } |
| |
| /* HPB version 1.0 is called as legacy version. */ |
| bool ufshpb_is_legacy(struct ufs_hba *hba) |
| { |
| return hba->ufshpb_dev.is_legacy; |
| } |
| |
| static struct ufshpb_lu *ufshpb_get_hpb_data(struct scsi_device *sdev) |
| { |
| return sdev->hostdata; |
| } |
| |
| static int ufshpb_get_state(struct ufshpb_lu *hpb) |
| { |
| return atomic_read(&hpb->hpb_state); |
| } |
| |
| static void ufshpb_set_state(struct ufshpb_lu *hpb, int state) |
| { |
| atomic_set(&hpb->hpb_state, state); |
| } |
| |
| static int ufshpb_is_valid_srgn(struct ufshpb_region *rgn, |
| struct ufshpb_subregion *srgn) |
| { |
| return rgn->rgn_state != HPB_RGN_INACTIVE && |
| srgn->srgn_state == HPB_SRGN_VALID; |
| } |
| |
| static bool ufshpb_is_read_cmd(struct scsi_cmnd *cmd) |
| { |
| return req_op(scsi_cmd_to_rq(cmd)) == REQ_OP_READ; |
| } |
| |
| static bool ufshpb_is_write_or_discard(struct scsi_cmnd *cmd) |
| { |
| return op_is_write(req_op(scsi_cmd_to_rq(cmd))) || |
| op_is_discard(req_op(scsi_cmd_to_rq(cmd))); |
| } |
| |
| static bool ufshpb_is_supported_chunk(struct ufshpb_lu *hpb, int transfer_len) |
| { |
| return transfer_len <= hpb->pre_req_max_tr_len; |
| } |
| |
| static bool ufshpb_is_general_lun(int lun) |
| { |
| return lun < UFS_UPIU_MAX_UNIT_NUM_ID; |
| } |
| |
| static bool ufshpb_is_pinned_region(struct ufshpb_lu *hpb, int rgn_idx) |
| { |
| return hpb->lu_pinned_end != PINNED_NOT_SET && |
| rgn_idx >= hpb->lu_pinned_start && rgn_idx <= hpb->lu_pinned_end; |
| } |
| |
| static void ufshpb_kick_map_work(struct ufshpb_lu *hpb) |
| { |
| bool ret = false; |
| unsigned long flags; |
| |
| if (ufshpb_get_state(hpb) != HPB_PRESENT) |
| return; |
| |
| spin_lock_irqsave(&hpb->rsp_list_lock, flags); |
| if (!list_empty(&hpb->lh_inact_rgn) || !list_empty(&hpb->lh_act_srgn)) |
| ret = true; |
| spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); |
| |
| if (ret) |
| queue_work(ufshpb_wq, &hpb->map_work); |
| } |
| |
| static bool ufshpb_is_hpb_rsp_valid(struct ufs_hba *hba, |
| struct ufshcd_lrb *lrbp, |
| struct utp_hpb_rsp *rsp_field) |
| { |
| /* Check HPB_UPDATE_ALERT */ |
| if (!(lrbp->ucd_rsp_ptr->header.dword_2 & |
| UPIU_HEADER_DWORD(0, 2, 0, 0))) |
| return false; |
| |
| if (be16_to_cpu(rsp_field->sense_data_len) != DEV_SENSE_SEG_LEN || |
| rsp_field->desc_type != DEV_DES_TYPE || |
| rsp_field->additional_len != DEV_ADDITIONAL_LEN || |
| rsp_field->active_rgn_cnt > MAX_ACTIVE_NUM || |
| rsp_field->inactive_rgn_cnt > MAX_INACTIVE_NUM || |
| rsp_field->hpb_op == HPB_RSP_NONE || |
| (rsp_field->hpb_op == HPB_RSP_REQ_REGION_UPDATE && |
| !rsp_field->active_rgn_cnt && !rsp_field->inactive_rgn_cnt)) |
| return false; |
| |
| if (!ufshpb_is_general_lun(rsp_field->lun)) { |
| dev_warn(hba->dev, "ufshpb: lun(%d) not supported\n", |
| lrbp->lun); |
| return false; |
| } |
| |
| return true; |
| } |
| |
| static void ufshpb_iterate_rgn(struct ufshpb_lu *hpb, int rgn_idx, int srgn_idx, |
| int srgn_offset, int cnt, bool set_dirty) |
| { |
| struct ufshpb_region *rgn; |
| struct ufshpb_subregion *srgn, *prev_srgn = NULL; |
| int set_bit_len; |
| int bitmap_len; |
| unsigned long flags; |
| |
| next_srgn: |
| rgn = hpb->rgn_tbl + rgn_idx; |
| srgn = rgn->srgn_tbl + srgn_idx; |
| |
| if (likely(!srgn->is_last)) |
| bitmap_len = hpb->entries_per_srgn; |
| else |
| bitmap_len = hpb->last_srgn_entries; |
| |
| if ((srgn_offset + cnt) > bitmap_len) |
| set_bit_len = bitmap_len - srgn_offset; |
| else |
| set_bit_len = cnt; |
| |
| spin_lock_irqsave(&hpb->rgn_state_lock, flags); |
| if (rgn->rgn_state != HPB_RGN_INACTIVE) { |
| if (set_dirty) { |
| if (srgn->srgn_state == HPB_SRGN_VALID) |
| bitmap_set(srgn->mctx->ppn_dirty, srgn_offset, |
| set_bit_len); |
| } else if (hpb->is_hcm) { |
| /* rewind the read timer for lru regions */ |
| rgn->read_timeout = ktime_add_ms(ktime_get(), |
| rgn->hpb->params.read_timeout_ms); |
| rgn->read_timeout_expiries = |
| rgn->hpb->params.read_timeout_expiries; |
| } |
| } |
| spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); |
| |
| if (hpb->is_hcm && prev_srgn != srgn) { |
| bool activate = false; |
| |
| spin_lock(&rgn->rgn_lock); |
| if (set_dirty) { |
| rgn->reads -= srgn->reads; |
| srgn->reads = 0; |
| set_bit(RGN_FLAG_DIRTY, &rgn->rgn_flags); |
| } else { |
| srgn->reads++; |
| rgn->reads++; |
| if (srgn->reads == hpb->params.activation_thld) |
| activate = true; |
| } |
| spin_unlock(&rgn->rgn_lock); |
| |
| if (activate || |
| test_and_clear_bit(RGN_FLAG_UPDATE, &rgn->rgn_flags)) { |
| spin_lock_irqsave(&hpb->rsp_list_lock, flags); |
| ufshpb_update_active_info(hpb, rgn_idx, srgn_idx); |
| spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); |
| dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, |
| "activate region %d-%d\n", rgn_idx, srgn_idx); |
| } |
| |
| prev_srgn = srgn; |
| } |
| |
| srgn_offset = 0; |
| if (++srgn_idx == hpb->srgns_per_rgn) { |
| srgn_idx = 0; |
| rgn_idx++; |
| } |
| |
| cnt -= set_bit_len; |
| if (cnt > 0) |
| goto next_srgn; |
| } |
| |
| static bool ufshpb_test_ppn_dirty(struct ufshpb_lu *hpb, int rgn_idx, |
| int srgn_idx, int srgn_offset, int cnt) |
| { |
| struct ufshpb_region *rgn; |
| struct ufshpb_subregion *srgn; |
| int bitmap_len; |
| int bit_len; |
| |
| next_srgn: |
| rgn = hpb->rgn_tbl + rgn_idx; |
| srgn = rgn->srgn_tbl + srgn_idx; |
| |
| if (likely(!srgn->is_last)) |
| bitmap_len = hpb->entries_per_srgn; |
| else |
| bitmap_len = hpb->last_srgn_entries; |
| |
| if (!ufshpb_is_valid_srgn(rgn, srgn)) |
| return true; |
| |
| /* |
| * If the region state is active, mctx must be allocated. |
| * In this case, check whether the region is evicted or |
| * mctx allocation fail. |
| */ |
| if (unlikely(!srgn->mctx)) { |
| dev_err(&hpb->sdev_ufs_lu->sdev_dev, |
| "no mctx in region %d subregion %d.\n", |
| srgn->rgn_idx, srgn->srgn_idx); |
| return true; |
| } |
| |
| if ((srgn_offset + cnt) > bitmap_len) |
| bit_len = bitmap_len - srgn_offset; |
| else |
| bit_len = cnt; |
| |
| if (find_next_bit(srgn->mctx->ppn_dirty, bit_len + srgn_offset, |
| srgn_offset) < bit_len + srgn_offset) |
| return true; |
| |
| srgn_offset = 0; |
| if (++srgn_idx == hpb->srgns_per_rgn) { |
| srgn_idx = 0; |
| rgn_idx++; |
| } |
| |
| cnt -= bit_len; |
| if (cnt > 0) |
| goto next_srgn; |
| |
| return false; |
| } |
| |
| static inline bool is_rgn_dirty(struct ufshpb_region *rgn) |
| { |
| return test_bit(RGN_FLAG_DIRTY, &rgn->rgn_flags); |
| } |
| |
| static int ufshpb_fill_ppn_from_page(struct ufshpb_lu *hpb, |
| struct ufshpb_map_ctx *mctx, int pos, |
| int len, __be64 *ppn_buf) |
| { |
| struct page *page; |
| int index, offset; |
| int copied; |
| |
| index = pos / (PAGE_SIZE / HPB_ENTRY_SIZE); |
| offset = pos % (PAGE_SIZE / HPB_ENTRY_SIZE); |
| |
| if ((offset + len) <= (PAGE_SIZE / HPB_ENTRY_SIZE)) |
| copied = len; |
| else |
| copied = (PAGE_SIZE / HPB_ENTRY_SIZE) - offset; |
| |
| page = mctx->m_page[index]; |
| if (unlikely(!page)) { |
| dev_err(&hpb->sdev_ufs_lu->sdev_dev, |
| "error. cannot find page in mctx\n"); |
| return -ENOMEM; |
| } |
| |
| memcpy(ppn_buf, page_address(page) + (offset * HPB_ENTRY_SIZE), |
| copied * HPB_ENTRY_SIZE); |
| |
| return copied; |
| } |
| |
| static void |
| ufshpb_get_pos_from_lpn(struct ufshpb_lu *hpb, unsigned long lpn, int *rgn_idx, |
| int *srgn_idx, int *offset) |
| { |
| int rgn_offset; |
| |
| *rgn_idx = lpn >> hpb->entries_per_rgn_shift; |
| rgn_offset = lpn & hpb->entries_per_rgn_mask; |
| *srgn_idx = rgn_offset >> hpb->entries_per_srgn_shift; |
| *offset = rgn_offset & hpb->entries_per_srgn_mask; |
| } |
| |
| static void |
| ufshpb_set_hpb_read_to_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, |
| __be64 ppn, u8 transfer_len) |
| { |
| unsigned char *cdb = lrbp->cmd->cmnd; |
| __be64 ppn_tmp = ppn; |
| cdb[0] = UFSHPB_READ; |
| |
| if (hba->dev_quirks & UFS_DEVICE_QUIRK_SWAP_L2P_ENTRY_FOR_HPB_READ) |
| ppn_tmp = (__force __be64)swab64((__force u64)ppn); |
| |
| /* ppn value is stored as big-endian in the host memory */ |
| memcpy(&cdb[6], &ppn_tmp, sizeof(__be64)); |
| cdb[14] = transfer_len; |
| cdb[15] = 0; |
| |
| lrbp->cmd->cmd_len = UFS_CDB_SIZE; |
| } |
| |
| /* |
| * This function will set up HPB read command using host-side L2P map data. |
| */ |
| int ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) |
| { |
| struct ufshpb_lu *hpb; |
| struct ufshpb_region *rgn; |
| struct ufshpb_subregion *srgn; |
| struct scsi_cmnd *cmd = lrbp->cmd; |
| u32 lpn; |
| __be64 ppn; |
| unsigned long flags; |
| int transfer_len, rgn_idx, srgn_idx, srgn_offset; |
| int err = 0; |
| |
| hpb = ufshpb_get_hpb_data(cmd->device); |
| if (!hpb) |
| return -ENODEV; |
| |
| if (ufshpb_get_state(hpb) == HPB_INIT) |
| return -ENODEV; |
| |
| if (ufshpb_get_state(hpb) != HPB_PRESENT) { |
| dev_notice(&hpb->sdev_ufs_lu->sdev_dev, |
| "%s: ufshpb state is not PRESENT", __func__); |
| return -ENODEV; |
| } |
| |
| if (blk_rq_is_passthrough(scsi_cmd_to_rq(cmd)) || |
| (!ufshpb_is_write_or_discard(cmd) && |
| !ufshpb_is_read_cmd(cmd))) |
| return 0; |
| |
| transfer_len = sectors_to_logical(cmd->device, |
| blk_rq_sectors(scsi_cmd_to_rq(cmd))); |
| if (unlikely(!transfer_len)) |
| return 0; |
| |
| lpn = sectors_to_logical(cmd->device, blk_rq_pos(scsi_cmd_to_rq(cmd))); |
| ufshpb_get_pos_from_lpn(hpb, lpn, &rgn_idx, &srgn_idx, &srgn_offset); |
| rgn = hpb->rgn_tbl + rgn_idx; |
| srgn = rgn->srgn_tbl + srgn_idx; |
| |
| /* If command type is WRITE or DISCARD, set bitmap as drity */ |
| if (ufshpb_is_write_or_discard(cmd)) { |
| ufshpb_iterate_rgn(hpb, rgn_idx, srgn_idx, srgn_offset, |
| transfer_len, true); |
| return 0; |
| } |
| |
| if (!ufshpb_is_supported_chunk(hpb, transfer_len)) |
| return 0; |
| |
| if (hpb->is_hcm) { |
| /* |
| * in host control mode, reads are the main source for |
| * activation trials. |
| */ |
| ufshpb_iterate_rgn(hpb, rgn_idx, srgn_idx, srgn_offset, |
| transfer_len, false); |
| |
| /* keep those counters normalized */ |
| if (rgn->reads > hpb->entries_per_srgn) |
| schedule_work(&hpb->ufshpb_normalization_work); |
| } |
| |
| spin_lock_irqsave(&hpb->rgn_state_lock, flags); |
| if (ufshpb_test_ppn_dirty(hpb, rgn_idx, srgn_idx, srgn_offset, |
| transfer_len)) { |
| hpb->stats.miss_cnt++; |
| spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); |
| return 0; |
| } |
| |
| err = ufshpb_fill_ppn_from_page(hpb, srgn->mctx, srgn_offset, 1, &ppn); |
| spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); |
| if (unlikely(err < 0)) { |
| /* |
| * In this case, the region state is active, |
| * but the ppn table is not allocated. |
| * Make sure that ppn table must be allocated on |
| * active state. |
| */ |
| dev_err(hba->dev, "get ppn failed. err %d\n", err); |
| return err; |
| } |
| |
| ufshpb_set_hpb_read_to_upiu(hba, lrbp, ppn, transfer_len); |
| |
| hpb->stats.hit_cnt++; |
| return 0; |
| } |
| |
| static struct ufshpb_req *ufshpb_get_req(struct ufshpb_lu *hpb, int rgn_idx, |
| enum req_op op, bool atomic) |
| { |
| struct ufshpb_req *rq; |
| struct request *req; |
| int retries = HPB_MAP_REQ_RETRIES; |
| |
| rq = kmem_cache_alloc(hpb->map_req_cache, GFP_KERNEL); |
| if (!rq) |
| return NULL; |
| |
| retry: |
| req = blk_mq_alloc_request(hpb->sdev_ufs_lu->request_queue, op, |
| BLK_MQ_REQ_NOWAIT); |
| |
| if (!atomic && (PTR_ERR(req) == -EWOULDBLOCK) && (--retries > 0)) { |
| usleep_range(3000, 3100); |
| goto retry; |
| } |
| |
| if (IS_ERR(req)) |
| goto free_rq; |
| |
| rq->hpb = hpb; |
| rq->req = req; |
| rq->rb.rgn_idx = rgn_idx; |
| |
| return rq; |
| |
| free_rq: |
| kmem_cache_free(hpb->map_req_cache, rq); |
| return NULL; |
| } |
| |
| static void ufshpb_put_req(struct ufshpb_lu *hpb, struct ufshpb_req *rq) |
| { |
| blk_mq_free_request(rq->req); |
| kmem_cache_free(hpb->map_req_cache, rq); |
| } |
| |
| static struct ufshpb_req *ufshpb_get_map_req(struct ufshpb_lu *hpb, |
| struct ufshpb_subregion *srgn) |
| { |
| struct ufshpb_req *map_req; |
| struct bio *bio; |
| unsigned long flags; |
| |
| if (hpb->is_hcm && |
| hpb->num_inflight_map_req >= hpb->params.inflight_map_req) { |
| dev_info(&hpb->sdev_ufs_lu->sdev_dev, |
| "map_req throttle. inflight %d throttle %d", |
| hpb->num_inflight_map_req, |
| hpb->params.inflight_map_req); |
| return NULL; |
| } |
| |
| map_req = ufshpb_get_req(hpb, srgn->rgn_idx, REQ_OP_DRV_IN, false); |
| if (!map_req) |
| return NULL; |
| |
| bio = bio_alloc(NULL, hpb->pages_per_srgn, 0, GFP_KERNEL); |
| if (!bio) { |
| ufshpb_put_req(hpb, map_req); |
| return NULL; |
| } |
| |
| map_req->bio = bio; |
| |
| map_req->rb.srgn_idx = srgn->srgn_idx; |
| map_req->rb.mctx = srgn->mctx; |
| |
| spin_lock_irqsave(&hpb->param_lock, flags); |
| hpb->num_inflight_map_req++; |
| spin_unlock_irqrestore(&hpb->param_lock, flags); |
| |
| return map_req; |
| } |
| |
| static void ufshpb_put_map_req(struct ufshpb_lu *hpb, |
| struct ufshpb_req *map_req) |
| { |
| unsigned long flags; |
| |
| bio_put(map_req->bio); |
| ufshpb_put_req(hpb, map_req); |
| |
| spin_lock_irqsave(&hpb->param_lock, flags); |
| hpb->num_inflight_map_req--; |
| spin_unlock_irqrestore(&hpb->param_lock, flags); |
| } |
| |
| static int ufshpb_clear_dirty_bitmap(struct ufshpb_lu *hpb, |
| struct ufshpb_subregion *srgn) |
| { |
| struct ufshpb_region *rgn; |
| u32 num_entries = hpb->entries_per_srgn; |
| |
| if (!srgn->mctx) { |
| dev_err(&hpb->sdev_ufs_lu->sdev_dev, |
| "no mctx in region %d subregion %d.\n", |
| srgn->rgn_idx, srgn->srgn_idx); |
| return -1; |
| } |
| |
| if (unlikely(srgn->is_last)) |
| num_entries = hpb->last_srgn_entries; |
| |
| bitmap_zero(srgn->mctx->ppn_dirty, num_entries); |
| |
| rgn = hpb->rgn_tbl + srgn->rgn_idx; |
| clear_bit(RGN_FLAG_DIRTY, &rgn->rgn_flags); |
| |
| return 0; |
| } |
| |
| static void ufshpb_update_active_info(struct ufshpb_lu *hpb, int rgn_idx, |
| int srgn_idx) |
| { |
| struct ufshpb_region *rgn; |
| struct ufshpb_subregion *srgn; |
| |
| rgn = hpb->rgn_tbl + rgn_idx; |
| srgn = rgn->srgn_tbl + srgn_idx; |
| |
| list_del_init(&rgn->list_inact_rgn); |
| |
| if (list_empty(&srgn->list_act_srgn)) |
| list_add_tail(&srgn->list_act_srgn, &hpb->lh_act_srgn); |
| |
| hpb->stats.rcmd_active_cnt++; |
| } |
| |
| static void ufshpb_update_inactive_info(struct ufshpb_lu *hpb, int rgn_idx) |
| { |
| struct ufshpb_region *rgn; |
| struct ufshpb_subregion *srgn; |
| int srgn_idx; |
| |
| rgn = hpb->rgn_tbl + rgn_idx; |
| |
| for_each_sub_region(rgn, srgn_idx, srgn) |
| list_del_init(&srgn->list_act_srgn); |
| |
| if (list_empty(&rgn->list_inact_rgn)) |
| list_add_tail(&rgn->list_inact_rgn, &hpb->lh_inact_rgn); |
| |
| hpb->stats.rcmd_inactive_cnt++; |
| } |
| |
| static void ufshpb_activate_subregion(struct ufshpb_lu *hpb, |
| struct ufshpb_subregion *srgn) |
| { |
| struct ufshpb_region *rgn; |
| |
| /* |
| * If there is no mctx in subregion |
| * after I/O progress for HPB_READ_BUFFER, the region to which the |
| * subregion belongs was evicted. |
| * Make sure the region must not evict in I/O progress |
| */ |
| if (!srgn->mctx) { |
| dev_err(&hpb->sdev_ufs_lu->sdev_dev, |
| "no mctx in region %d subregion %d.\n", |
| srgn->rgn_idx, srgn->srgn_idx); |
| srgn->srgn_state = HPB_SRGN_INVALID; |
| return; |
| } |
| |
| rgn = hpb->rgn_tbl + srgn->rgn_idx; |
| |
| if (unlikely(rgn->rgn_state == HPB_RGN_INACTIVE)) { |
| dev_err(&hpb->sdev_ufs_lu->sdev_dev, |
| "region %d subregion %d evicted\n", |
| srgn->rgn_idx, srgn->srgn_idx); |
| srgn->srgn_state = HPB_SRGN_INVALID; |
| return; |
| } |
| srgn->srgn_state = HPB_SRGN_VALID; |
| } |
| |
| static void ufshpb_umap_req_compl_fn(struct request *req, blk_status_t error) |
| { |
| struct ufshpb_req *umap_req = (struct ufshpb_req *)req->end_io_data; |
| |
| ufshpb_put_req(umap_req->hpb, umap_req); |
| } |
| |
| static void ufshpb_map_req_compl_fn(struct request *req, blk_status_t error) |
| { |
| struct ufshpb_req *map_req = (struct ufshpb_req *) req->end_io_data; |
| struct ufshpb_lu *hpb = map_req->hpb; |
| struct ufshpb_subregion *srgn; |
| unsigned long flags; |
| |
| srgn = hpb->rgn_tbl[map_req->rb.rgn_idx].srgn_tbl + |
| map_req->rb.srgn_idx; |
| |
| ufshpb_clear_dirty_bitmap(hpb, srgn); |
| spin_lock_irqsave(&hpb->rgn_state_lock, flags); |
| ufshpb_activate_subregion(hpb, srgn); |
| spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); |
| |
| ufshpb_put_map_req(map_req->hpb, map_req); |
| } |
| |
| static void ufshpb_set_unmap_cmd(unsigned char *cdb, struct ufshpb_region *rgn) |
| { |
| cdb[0] = UFSHPB_WRITE_BUFFER; |
| cdb[1] = rgn ? UFSHPB_WRITE_BUFFER_INACT_SINGLE_ID : |
| UFSHPB_WRITE_BUFFER_INACT_ALL_ID; |
| if (rgn) |
| put_unaligned_be16(rgn->rgn_idx, &cdb[2]); |
| cdb[9] = 0x00; |
| } |
| |
| static void ufshpb_set_read_buf_cmd(unsigned char *cdb, int rgn_idx, |
| int srgn_idx, int srgn_mem_size) |
| { |
| cdb[0] = UFSHPB_READ_BUFFER; |
| cdb[1] = UFSHPB_READ_BUFFER_ID; |
| |
| put_unaligned_be16(rgn_idx, &cdb[2]); |
| put_unaligned_be16(srgn_idx, &cdb[4]); |
| put_unaligned_be24(srgn_mem_size, &cdb[6]); |
| |
| cdb[9] = 0x00; |
| } |
| |
| static void ufshpb_execute_umap_req(struct ufshpb_lu *hpb, |
| struct ufshpb_req *umap_req, |
| struct ufshpb_region *rgn) |
| { |
| struct request *req = umap_req->req; |
| struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(req); |
| |
| req->timeout = 0; |
| req->end_io_data = umap_req; |
| req->end_io = ufshpb_umap_req_compl_fn; |
| |
| ufshpb_set_unmap_cmd(scmd->cmnd, rgn); |
| scmd->cmd_len = HPB_WRITE_BUFFER_CMD_LENGTH; |
| |
| blk_execute_rq_nowait(req, true); |
| |
| hpb->stats.umap_req_cnt++; |
| } |
| |
| static int ufshpb_execute_map_req(struct ufshpb_lu *hpb, |
| struct ufshpb_req *map_req, bool last) |
| { |
| struct request_queue *q; |
| struct request *req; |
| struct scsi_cmnd *scmd; |
| int mem_size = hpb->srgn_mem_size; |
| int ret = 0; |
| int i; |
| |
| q = hpb->sdev_ufs_lu->request_queue; |
| for (i = 0; i < hpb->pages_per_srgn; i++) { |
| ret = bio_add_pc_page(q, map_req->bio, map_req->rb.mctx->m_page[i], |
| PAGE_SIZE, 0); |
| if (ret != PAGE_SIZE) { |
| dev_err(&hpb->sdev_ufs_lu->sdev_dev, |
| "bio_add_pc_page fail %d - %d\n", |
| map_req->rb.rgn_idx, map_req->rb.srgn_idx); |
| return ret; |
| } |
| } |
| |
| req = map_req->req; |
| |
| blk_rq_append_bio(req, map_req->bio); |
| |
| req->end_io_data = map_req; |
| req->end_io = ufshpb_map_req_compl_fn; |
| |
| if (unlikely(last)) |
| mem_size = hpb->last_srgn_entries * HPB_ENTRY_SIZE; |
| |
| scmd = blk_mq_rq_to_pdu(req); |
| ufshpb_set_read_buf_cmd(scmd->cmnd, map_req->rb.rgn_idx, |
| map_req->rb.srgn_idx, mem_size); |
| scmd->cmd_len = HPB_READ_BUFFER_CMD_LENGTH; |
| |
| blk_execute_rq_nowait(req, true); |
| |
| hpb->stats.map_req_cnt++; |
| return 0; |
| } |
| |
| static struct ufshpb_map_ctx *ufshpb_get_map_ctx(struct ufshpb_lu *hpb, |
| bool last) |
| { |
| struct ufshpb_map_ctx *mctx; |
| u32 num_entries = hpb->entries_per_srgn; |
| int i, j; |
| |
| mctx = mempool_alloc(ufshpb_mctx_pool, GFP_KERNEL); |
| if (!mctx) |
| return NULL; |
| |
| mctx->m_page = kmem_cache_alloc(hpb->m_page_cache, GFP_KERNEL); |
| if (!mctx->m_page) |
| goto release_mctx; |
| |
| if (unlikely(last)) |
| num_entries = hpb->last_srgn_entries; |
| |
| mctx->ppn_dirty = bitmap_zalloc(num_entries, GFP_KERNEL); |
| if (!mctx->ppn_dirty) |
| goto release_m_page; |
| |
| for (i = 0; i < hpb->pages_per_srgn; i++) { |
| mctx->m_page[i] = mempool_alloc(ufshpb_page_pool, GFP_KERNEL); |
| if (!mctx->m_page[i]) { |
| for (j = 0; j < i; j++) |
| mempool_free(mctx->m_page[j], ufshpb_page_pool); |
| goto release_ppn_dirty; |
| } |
| clear_page(page_address(mctx->m_page[i])); |
| } |
| |
| return mctx; |
| |
| release_ppn_dirty: |
| bitmap_free(mctx->ppn_dirty); |
| release_m_page: |
| kmem_cache_free(hpb->m_page_cache, mctx->m_page); |
| release_mctx: |
| mempool_free(mctx, ufshpb_mctx_pool); |
| return NULL; |
| } |
| |
| static void ufshpb_put_map_ctx(struct ufshpb_lu *hpb, |
| struct ufshpb_map_ctx *mctx) |
| { |
| int i; |
| |
| for (i = 0; i < hpb->pages_per_srgn; i++) |
| mempool_free(mctx->m_page[i], ufshpb_page_pool); |
| |
| bitmap_free(mctx->ppn_dirty); |
| kmem_cache_free(hpb->m_page_cache, mctx->m_page); |
| mempool_free(mctx, ufshpb_mctx_pool); |
| } |
| |
| static int ufshpb_check_srgns_issue_state(struct ufshpb_lu *hpb, |
| struct ufshpb_region *rgn) |
| { |
| struct ufshpb_subregion *srgn; |
| int srgn_idx; |
| |
| for_each_sub_region(rgn, srgn_idx, srgn) |
| if (srgn->srgn_state == HPB_SRGN_ISSUED) |
| return -EPERM; |
| |
| return 0; |
| } |
| |
| static void ufshpb_read_to_handler(struct work_struct *work) |
| { |
| struct ufshpb_lu *hpb = container_of(work, struct ufshpb_lu, |
| ufshpb_read_to_work.work); |
| struct victim_select_info *lru_info = &hpb->lru_info; |
| struct ufshpb_region *rgn, *next_rgn; |
| unsigned long flags; |
| unsigned int poll; |
| LIST_HEAD(expired_list); |
| |
| if (test_and_set_bit(TIMEOUT_WORK_RUNNING, &hpb->work_data_bits)) |
| return; |
| |
| spin_lock_irqsave(&hpb->rgn_state_lock, flags); |
| |
| list_for_each_entry_safe(rgn, next_rgn, &lru_info->lh_lru_rgn, |
| list_lru_rgn) { |
| bool timedout = ktime_after(ktime_get(), rgn->read_timeout); |
| |
| if (timedout) { |
| rgn->read_timeout_expiries--; |
| if (is_rgn_dirty(rgn) || |
| rgn->read_timeout_expiries == 0) |
| list_add(&rgn->list_expired_rgn, &expired_list); |
| else |
| rgn->read_timeout = ktime_add_ms(ktime_get(), |
| hpb->params.read_timeout_ms); |
| } |
| } |
| |
| spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); |
| |
| list_for_each_entry_safe(rgn, next_rgn, &expired_list, |
| list_expired_rgn) { |
| list_del_init(&rgn->list_expired_rgn); |
| spin_lock_irqsave(&hpb->rsp_list_lock, flags); |
| ufshpb_update_inactive_info(hpb, rgn->rgn_idx); |
| spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); |
| } |
| |
| ufshpb_kick_map_work(hpb); |
| |
| clear_bit(TIMEOUT_WORK_RUNNING, &hpb->work_data_bits); |
| |
| poll = hpb->params.timeout_polling_interval_ms; |
| schedule_delayed_work(&hpb->ufshpb_read_to_work, |
| msecs_to_jiffies(poll)); |
| } |
| |
| static void ufshpb_add_lru_info(struct victim_select_info *lru_info, |
| struct ufshpb_region *rgn) |
| { |
| rgn->rgn_state = HPB_RGN_ACTIVE; |
| list_add_tail(&rgn->list_lru_rgn, &lru_info->lh_lru_rgn); |
| atomic_inc(&lru_info->active_cnt); |
| if (rgn->hpb->is_hcm) { |
| rgn->read_timeout = |
| ktime_add_ms(ktime_get(), |
| rgn->hpb->params.read_timeout_ms); |
| rgn->read_timeout_expiries = |
| rgn->hpb->params.read_timeout_expiries; |
| } |
| } |
| |
| static void ufshpb_hit_lru_info(struct victim_select_info *lru_info, |
| struct ufshpb_region *rgn) |
| { |
| list_move_tail(&rgn->list_lru_rgn, &lru_info->lh_lru_rgn); |
| } |
| |
| static struct ufshpb_region *ufshpb_victim_lru_info(struct ufshpb_lu *hpb) |
| { |
| struct victim_select_info *lru_info = &hpb->lru_info; |
| struct ufshpb_region *rgn, *victim_rgn = NULL; |
| |
| list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) { |
| if (ufshpb_check_srgns_issue_state(hpb, rgn)) |
| continue; |
| |
| /* |
| * in host control mode, verify that the exiting region |
| * has fewer reads |
| */ |
| if (hpb->is_hcm && |
| rgn->reads > hpb->params.eviction_thld_exit) |
| continue; |
| |
| victim_rgn = rgn; |
| break; |
| } |
| |
| if (!victim_rgn) |
| dev_err(&hpb->sdev_ufs_lu->sdev_dev, |
| "%s: no region allocated\n", |
| __func__); |
| |
| return victim_rgn; |
| } |
| |
| static void ufshpb_cleanup_lru_info(struct victim_select_info *lru_info, |
| struct ufshpb_region *rgn) |
| { |
| list_del_init(&rgn->list_lru_rgn); |
| rgn->rgn_state = HPB_RGN_INACTIVE; |
| atomic_dec(&lru_info->active_cnt); |
| } |
| |
| static void ufshpb_purge_active_subregion(struct ufshpb_lu *hpb, |
| struct ufshpb_subregion *srgn) |
| { |
| if (srgn->srgn_state != HPB_SRGN_UNUSED) { |
| ufshpb_put_map_ctx(hpb, srgn->mctx); |
| srgn->srgn_state = HPB_SRGN_UNUSED; |
| srgn->mctx = NULL; |
| } |
| } |
| |
| static int ufshpb_issue_umap_req(struct ufshpb_lu *hpb, |
| struct ufshpb_region *rgn, |
| bool atomic) |
| { |
| struct ufshpb_req *umap_req; |
| int rgn_idx = rgn ? rgn->rgn_idx : 0; |
| |
| umap_req = ufshpb_get_req(hpb, rgn_idx, REQ_OP_DRV_OUT, atomic); |
| if (!umap_req) |
| return -ENOMEM; |
| |
| ufshpb_execute_umap_req(hpb, umap_req, rgn); |
| |
| return 0; |
| } |
| |
| static int ufshpb_issue_umap_single_req(struct ufshpb_lu *hpb, |
| struct ufshpb_region *rgn) |
| { |
| return ufshpb_issue_umap_req(hpb, rgn, true); |
| } |
| |
| static void __ufshpb_evict_region(struct ufshpb_lu *hpb, |
| struct ufshpb_region *rgn) |
| { |
| struct victim_select_info *lru_info; |
| struct ufshpb_subregion *srgn; |
| int srgn_idx; |
| |
| lru_info = &hpb->lru_info; |
| |
| dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, "evict region %d\n", rgn->rgn_idx); |
| |
| ufshpb_cleanup_lru_info(lru_info, rgn); |
| |
| for_each_sub_region(rgn, srgn_idx, srgn) |
| ufshpb_purge_active_subregion(hpb, srgn); |
| } |
| |
| static int ufshpb_evict_region(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) |
| { |
| unsigned long flags; |
| int ret = 0; |
| |
| spin_lock_irqsave(&hpb->rgn_state_lock, flags); |
| if (rgn->rgn_state == HPB_RGN_PINNED) { |
| dev_warn(&hpb->sdev_ufs_lu->sdev_dev, |
| "pinned region cannot drop-out. region %d\n", |
| rgn->rgn_idx); |
| goto out; |
| } |
| |
| if (!list_empty(&rgn->list_lru_rgn)) { |
| if (ufshpb_check_srgns_issue_state(hpb, rgn)) { |
| ret = -EBUSY; |
| goto out; |
| } |
| |
| if (hpb->is_hcm) { |
| spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); |
| ret = ufshpb_issue_umap_single_req(hpb, rgn); |
| spin_lock_irqsave(&hpb->rgn_state_lock, flags); |
| if (ret) |
| goto out; |
| } |
| |
| __ufshpb_evict_region(hpb, rgn); |
| } |
| out: |
| spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); |
| return ret; |
| } |
| |
| static int ufshpb_issue_map_req(struct ufshpb_lu *hpb, |
| struct ufshpb_region *rgn, |
| struct ufshpb_subregion *srgn) |
| { |
| struct ufshpb_req *map_req; |
| unsigned long flags; |
| int ret; |
| int err = -EAGAIN; |
| bool alloc_required = false; |
| enum HPB_SRGN_STATE state = HPB_SRGN_INVALID; |
| |
| spin_lock_irqsave(&hpb->rgn_state_lock, flags); |
| |
| if (ufshpb_get_state(hpb) != HPB_PRESENT) { |
| dev_notice(&hpb->sdev_ufs_lu->sdev_dev, |
| "%s: ufshpb state is not PRESENT\n", __func__); |
| goto unlock_out; |
| } |
| |
| if ((rgn->rgn_state == HPB_RGN_INACTIVE) && |
| (srgn->srgn_state == HPB_SRGN_INVALID)) { |
| err = 0; |
| goto unlock_out; |
| } |
| |
| if (srgn->srgn_state == HPB_SRGN_UNUSED) |
| alloc_required = true; |
| |
| /* |
| * If the subregion is already ISSUED state, |
| * a specific event (e.g., GC or wear-leveling, etc.) occurs in |
| * the device and HPB response for map loading is received. |
| * In this case, after finishing the HPB_READ_BUFFER, |
| * the next HPB_READ_BUFFER is performed again to obtain the latest |
| * map data. |
| */ |
| if (srgn->srgn_state == HPB_SRGN_ISSUED) |
| goto unlock_out; |
| |
| srgn->srgn_state = HPB_SRGN_ISSUED; |
| spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); |
| |
| if (alloc_required) { |
| srgn->mctx = ufshpb_get_map_ctx(hpb, srgn->is_last); |
| if (!srgn->mctx) { |
| dev_err(&hpb->sdev_ufs_lu->sdev_dev, |
| "get map_ctx failed. region %d - %d\n", |
| rgn->rgn_idx, srgn->srgn_idx); |
| state = HPB_SRGN_UNUSED; |
| goto change_srgn_state; |
| } |
| } |
| |
| map_req = ufshpb_get_map_req(hpb, srgn); |
| if (!map_req) |
| goto change_srgn_state; |
| |
| |
| ret = ufshpb_execute_map_req(hpb, map_req, srgn->is_last); |
| if (ret) { |
| dev_err(&hpb->sdev_ufs_lu->sdev_dev, |
| "%s: issue map_req failed: %d, region %d - %d\n", |
| __func__, ret, srgn->rgn_idx, srgn->srgn_idx); |
| goto free_map_req; |
| } |
| return 0; |
| |
| free_map_req: |
| ufshpb_put_map_req(hpb, map_req); |
| change_srgn_state: |
| spin_lock_irqsave(&hpb->rgn_state_lock, flags); |
| srgn->srgn_state = state; |
| unlock_out: |
| spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); |
| return err; |
| } |
| |
| static int ufshpb_add_region(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) |
| { |
| struct ufshpb_region *victim_rgn = NULL; |
| struct victim_select_info *lru_info = &hpb->lru_info; |
| unsigned long flags; |
| int ret = 0; |
| |
| spin_lock_irqsave(&hpb->rgn_state_lock, flags); |
| /* |
| * If region belongs to lru_list, just move the region |
| * to the front of lru list because the state of the region |
| * is already active-state. |
| */ |
| if (!list_empty(&rgn->list_lru_rgn)) { |
| ufshpb_hit_lru_info(lru_info, rgn); |
| goto out; |
| } |
| |
| if (rgn->rgn_state == HPB_RGN_INACTIVE) { |
| if (atomic_read(&lru_info->active_cnt) == |
| lru_info->max_lru_active_cnt) { |
| /* |
| * If the maximum number of active regions |
| * is exceeded, evict the least recently used region. |
| * This case may occur when the device responds |
| * to the eviction information late. |
| * It is okay to evict the least recently used region, |
| * because the device could detect this region |
| * by not issuing HPB_READ |
| * |
| * in host control mode, verify that the entering |
| * region has enough reads |
| */ |
| if (hpb->is_hcm && |
| rgn->reads < hpb->params.eviction_thld_enter) { |
| ret = -EACCES; |
| goto out; |
| } |
| |
| victim_rgn = ufshpb_victim_lru_info(hpb); |
| if (!victim_rgn) { |
| dev_warn(&hpb->sdev_ufs_lu->sdev_dev, |
| "cannot get victim region %s\n", |
| hpb->is_hcm ? "" : "error"); |
| ret = -ENOMEM; |
| goto out; |
| } |
| |
| dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, |
| "LRU full (%d), choose victim %d\n", |
| atomic_read(&lru_info->active_cnt), |
| victim_rgn->rgn_idx); |
| |
| if (hpb->is_hcm) { |
| spin_unlock_irqrestore(&hpb->rgn_state_lock, |
| flags); |
| ret = ufshpb_issue_umap_single_req(hpb, |
| victim_rgn); |
| spin_lock_irqsave(&hpb->rgn_state_lock, |
| flags); |
| if (ret) |
| goto out; |
| } |
| |
| __ufshpb_evict_region(hpb, victim_rgn); |
| } |
| |
| /* |
| * When a region is added to lru_info list_head, |
| * it is guaranteed that the subregion has been |
| * assigned all mctx. If failed, try to receive mctx again |
| * without being added to lru_info list_head |
| */ |
| ufshpb_add_lru_info(lru_info, rgn); |
| } |
| out: |
| spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); |
| return ret; |
| } |
| /** |
| *ufshpb_submit_region_inactive() - submit a region to be inactivated later |
| *@hpb: per-LU HPB instance |
| *@region_index: the index associated with the region that will be inactivated later |
| */ |
| static void ufshpb_submit_region_inactive(struct ufshpb_lu *hpb, int region_index) |
| { |
| int subregion_index; |
| struct ufshpb_region *rgn; |
| struct ufshpb_subregion *srgn; |
| |
| /* |
| * Remove this region from active region list and add it to inactive list |
| */ |
| spin_lock(&hpb->rsp_list_lock); |
| ufshpb_update_inactive_info(hpb, region_index); |
| spin_unlock(&hpb->rsp_list_lock); |
| |
| rgn = hpb->rgn_tbl + region_index; |
| |
| /* |
| * Set subregion state to be HPB_SRGN_INVALID, there will no HPB read on this subregion |
| */ |
| spin_lock(&hpb->rgn_state_lock); |
| if (rgn->rgn_state != HPB_RGN_INACTIVE) { |
| for (subregion_index = 0; subregion_index < rgn->srgn_cnt; subregion_index++) { |
| srgn = rgn->srgn_tbl + subregion_index; |
| if (srgn->srgn_state == HPB_SRGN_VALID) |
| srgn->srgn_state = HPB_SRGN_INVALID; |
| } |
| } |
| spin_unlock(&hpb->rgn_state_lock); |
| } |
| |
| static void ufshpb_rsp_req_region_update(struct ufshpb_lu *hpb, |
| struct utp_hpb_rsp *rsp_field) |
| { |
| struct ufshpb_region *rgn; |
| struct ufshpb_subregion *srgn; |
| int i, rgn_i, srgn_i; |
| |
| BUILD_BUG_ON(sizeof(struct ufshpb_active_field) != HPB_ACT_FIELD_SIZE); |
| /* |
| * If the active region and the inactive region are the same, |
| * we will inactivate this region. |
| * The device could check this (region inactivated) and |
| * will response the proper active region information |
| */ |
| for (i = 0; i < rsp_field->active_rgn_cnt; i++) { |
| rgn_i = |
| be16_to_cpu(rsp_field->hpb_active_field[i].active_rgn); |
| srgn_i = |
| be16_to_cpu(rsp_field->hpb_active_field[i].active_srgn); |
| |
| rgn = hpb->rgn_tbl + rgn_i; |
| if (hpb->is_hcm && |
| (rgn->rgn_state != HPB_RGN_ACTIVE || is_rgn_dirty(rgn))) { |
| /* |
| * in host control mode, subregion activation |
| * recommendations are only allowed to active regions. |
| * Also, ignore recommendations for dirty regions - the |
| * host will make decisions concerning those by himself |
| */ |
| continue; |
| } |
| |
| dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, |
| "activate(%d) region %d - %d\n", i, rgn_i, srgn_i); |
| |
| spin_lock(&hpb->rsp_list_lock); |
| ufshpb_update_active_info(hpb, rgn_i, srgn_i); |
| spin_unlock(&hpb->rsp_list_lock); |
| |
| srgn = rgn->srgn_tbl + srgn_i; |
| |
| /* blocking HPB_READ */ |
| spin_lock(&hpb->rgn_state_lock); |
| if (srgn->srgn_state == HPB_SRGN_VALID) |
| srgn->srgn_state = HPB_SRGN_INVALID; |
| spin_unlock(&hpb->rgn_state_lock); |
| } |
| |
| if (hpb->is_hcm) { |
| /* |
| * in host control mode the device is not allowed to inactivate |
| * regions |
| */ |
| goto out; |
| } |
| |
| for (i = 0; i < rsp_field->inactive_rgn_cnt; i++) { |
| rgn_i = be16_to_cpu(rsp_field->hpb_inactive_field[i]); |
| dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, "inactivate(%d) region %d\n", i, rgn_i); |
| ufshpb_submit_region_inactive(hpb, rgn_i); |
| } |
| |
| out: |
| dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, "Noti: #ACT %u #INACT %u\n", |
| rsp_field->active_rgn_cnt, rsp_field->inactive_rgn_cnt); |
| |
| if (ufshpb_get_state(hpb) == HPB_PRESENT) |
| queue_work(ufshpb_wq, &hpb->map_work); |
| } |
| |
| /* |
| * Set the flags of all active regions to RGN_FLAG_UPDATE to let host side reload L2P entries later |
| */ |
| static void ufshpb_set_regions_update(struct ufshpb_lu *hpb) |
| { |
| struct victim_select_info *lru_info = &hpb->lru_info; |
| struct ufshpb_region *rgn; |
| unsigned long flags; |
| |
| spin_lock_irqsave(&hpb->rgn_state_lock, flags); |
| |
| list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) |
| set_bit(RGN_FLAG_UPDATE, &rgn->rgn_flags); |
| |
| spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); |
| } |
| |
| static void ufshpb_dev_reset_handler(struct ufs_hba *hba) |
| { |
| struct scsi_device *sdev; |
| struct ufshpb_lu *hpb; |
| |
| __shost_for_each_device(sdev, hba->host) { |
| hpb = ufshpb_get_hpb_data(sdev); |
| if (!hpb) |
| continue; |
| |
| if (hpb->is_hcm) { |
| /* |
| * For the HPB host control mode, in case device powered up and lost HPB |
| * information, we will set the region flag to be RGN_FLAG_UPDATE, it will |
| * let host reload its L2P entries(reactivate region in the UFS device). |
| */ |
| ufshpb_set_regions_update(hpb); |
| } else { |
| /* |
| * For the HPB device control mode, if host side receives 02h:HPB Operation |
| * in UPIU response, which means device recommends the host side should |
| * inactivate all active regions. Here we add all active regions to inactive |
| * list, they will be inactivated later in ufshpb_map_work_handler(). |
| */ |
| struct victim_select_info *lru_info = &hpb->lru_info; |
| struct ufshpb_region *rgn; |
| |
| list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) |
| ufshpb_submit_region_inactive(hpb, rgn->rgn_idx); |
| |
| if (ufshpb_get_state(hpb) == HPB_PRESENT) |
| queue_work(ufshpb_wq, &hpb->map_work); |
| } |
| } |
| } |
| |
| /* |
| * This function will parse recommended active subregion information in sense |
| * data field of response UPIU with SAM_STAT_GOOD state. |
| */ |
| void ufshpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) |
| { |
| struct ufshpb_lu *hpb = ufshpb_get_hpb_data(lrbp->cmd->device); |
| struct utp_hpb_rsp *rsp_field = &lrbp->ucd_rsp_ptr->hr; |
| int data_seg_len; |
| |
| data_seg_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) |
| & MASK_RSP_UPIU_DATA_SEG_LEN; |
| |
| /* If data segment length is zero, rsp_field is not valid */ |
| if (!data_seg_len) |
| return; |
| |
| if (unlikely(lrbp->lun != rsp_field->lun)) { |
| struct scsi_device *sdev; |
| bool found = false; |
| |
| __shost_for_each_device(sdev, hba->host) { |
| hpb = ufshpb_get_hpb_data(sdev); |
| |
| if (!hpb) |
| continue; |
| |
| if (rsp_field->lun == hpb->lun) { |
| found = true; |
| break; |
| } |
| } |
| |
| if (!found) |
| return; |
| } |
| |
| if (!hpb) |
| return; |
| |
| if (ufshpb_get_state(hpb) == HPB_INIT) |
| return; |
| |
| if ((ufshpb_get_state(hpb) != HPB_PRESENT) && |
| (ufshpb_get_state(hpb) != HPB_SUSPEND)) { |
| dev_notice(&hpb->sdev_ufs_lu->sdev_dev, |
| "%s: ufshpb state is not PRESENT/SUSPEND\n", |
| __func__); |
| return; |
| } |
| |
| BUILD_BUG_ON(sizeof(struct utp_hpb_rsp) != UTP_HPB_RSP_SIZE); |
| |
| if (!ufshpb_is_hpb_rsp_valid(hba, lrbp, rsp_field)) |
| return; |
| |
| hpb->stats.rcmd_noti_cnt++; |
| |
| switch (rsp_field->hpb_op) { |
| case HPB_RSP_REQ_REGION_UPDATE: |
| if (data_seg_len != DEV_DATA_SEG_LEN) |
| dev_warn(&hpb->sdev_ufs_lu->sdev_dev, |
| "%s: data seg length is not same.\n", |
| __func__); |
| ufshpb_rsp_req_region_update(hpb, rsp_field); |
| break; |
| case HPB_RSP_DEV_RESET: |
| dev_warn(&hpb->sdev_ufs_lu->sdev_dev, |
| "UFS device lost HPB information during PM.\n"); |
| ufshpb_dev_reset_handler(hba); |
| |
| break; |
| default: |
| dev_notice(&hpb->sdev_ufs_lu->sdev_dev, |
| "hpb_op is not available: %d\n", |
| rsp_field->hpb_op); |
| break; |
| } |
| } |
| |
| static void ufshpb_add_active_list(struct ufshpb_lu *hpb, |
| struct ufshpb_region *rgn, |
| struct ufshpb_subregion *srgn) |
| { |
| if (!list_empty(&rgn->list_inact_rgn)) |
| return; |
| |
| if (!list_empty(&srgn->list_act_srgn)) { |
| list_move(&srgn->list_act_srgn, &hpb->lh_act_srgn); |
| return; |
| } |
| |
| list_add(&srgn->list_act_srgn, &hpb->lh_act_srgn); |
| } |
| |
| static void ufshpb_add_pending_evict_list(struct ufshpb_lu *hpb, |
| struct ufshpb_region *rgn, |
| struct list_head *pending_list) |
| { |
| struct ufshpb_subregion *srgn; |
| int srgn_idx; |
| |
| if (!list_empty(&rgn->list_inact_rgn)) |
| return; |
| |
| for_each_sub_region(rgn, srgn_idx, srgn) |
| if (!list_empty(&srgn->list_act_srgn)) |
| return; |
| |
| list_add_tail(&rgn->list_inact_rgn, pending_list); |
| } |
| |
| static void ufshpb_run_active_subregion_list(struct ufshpb_lu *hpb) |
| { |
| struct ufshpb_region *rgn; |
| struct ufshpb_subregion *srgn; |
| unsigned long flags; |
| int ret = 0; |
| |
| spin_lock_irqsave(&hpb->rsp_list_lock, flags); |
| while ((srgn = list_first_entry_or_null(&hpb->lh_act_srgn, |
| struct ufshpb_subregion, |
| list_act_srgn))) { |
| if (ufshpb_get_state(hpb) == HPB_SUSPEND) |
| break; |
| |
| list_del_init(&srgn->list_act_srgn); |
| spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); |
| |
| rgn = hpb->rgn_tbl + srgn->rgn_idx; |
| ret = ufshpb_add_region(hpb, rgn); |
| if (ret) |
| goto active_failed; |
| |
| ret = ufshpb_issue_map_req(hpb, rgn, srgn); |
| if (ret) { |
| dev_err(&hpb->sdev_ufs_lu->sdev_dev, |
| "issue map_req failed. ret %d, region %d - %d\n", |
| ret, rgn->rgn_idx, srgn->srgn_idx); |
| goto active_failed; |
| } |
| spin_lock_irqsave(&hpb->rsp_list_lock, flags); |
| } |
| spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); |
| return; |
| |
| active_failed: |
| dev_err(&hpb->sdev_ufs_lu->sdev_dev, "failed to activate region %d - %d, will retry\n", |
| rgn->rgn_idx, srgn->srgn_idx); |
| spin_lock_irqsave(&hpb->rsp_list_lock, flags); |
| ufshpb_add_active_list(hpb, rgn, srgn); |
| spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); |
| } |
| |
| static void ufshpb_run_inactive_region_list(struct ufshpb_lu *hpb) |
| { |
| struct ufshpb_region *rgn; |
| unsigned long flags; |
| int ret; |
| LIST_HEAD(pending_list); |
| |
| spin_lock_irqsave(&hpb->rsp_list_lock, flags); |
| while ((rgn = list_first_entry_or_null(&hpb->lh_inact_rgn, |
| struct ufshpb_region, |
| list_inact_rgn))) { |
| if (ufshpb_get_state(hpb) == HPB_SUSPEND) |
| break; |
| |
| list_del_init(&rgn->list_inact_rgn); |
| spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); |
| |
| ret = ufshpb_evict_region(hpb, rgn); |
| if (ret) { |
| spin_lock_irqsave(&hpb->rsp_list_lock, flags); |
| ufshpb_add_pending_evict_list(hpb, rgn, &pending_list); |
| spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); |
| } |
| |
| spin_lock_irqsave(&hpb->rsp_list_lock, flags); |
| } |
| |
| list_splice(&pending_list, &hpb->lh_inact_rgn); |
| spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); |
| } |
| |
| static void ufshpb_normalization_work_handler(struct work_struct *work) |
| { |
| struct ufshpb_lu *hpb = container_of(work, struct ufshpb_lu, |
| ufshpb_normalization_work); |
| int rgn_idx; |
| u8 factor = hpb->params.normalization_factor; |
| |
| for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) { |
| struct ufshpb_region *rgn = hpb->rgn_tbl + rgn_idx; |
| int srgn_idx; |
| |
| spin_lock(&rgn->rgn_lock); |
| rgn->reads = 0; |
| for (srgn_idx = 0; srgn_idx < hpb->srgns_per_rgn; srgn_idx++) { |
| struct ufshpb_subregion *srgn = rgn->srgn_tbl + srgn_idx; |
| |
| srgn->reads >>= factor; |
| rgn->reads += srgn->reads; |
| } |
| spin_unlock(&rgn->rgn_lock); |
| |
| if (rgn->rgn_state != HPB_RGN_ACTIVE || rgn->reads) |
| continue; |
| |
| /* if region is active but has no reads - inactivate it */ |
| spin_lock(&hpb->rsp_list_lock); |
| ufshpb_update_inactive_info(hpb, rgn->rgn_idx); |
| spin_unlock(&hpb->rsp_list_lock); |
| } |
| } |
| |
| static void ufshpb_map_work_handler(struct work_struct *work) |
| { |
| struct ufshpb_lu *hpb = container_of(work, struct ufshpb_lu, map_work); |
| |
| if (ufshpb_get_state(hpb) != HPB_PRESENT) { |
| dev_notice(&hpb->sdev_ufs_lu->sdev_dev, |
| "%s: ufshpb state is not PRESENT\n", __func__); |
| return; |
| } |
| |
| ufshpb_run_inactive_region_list(hpb); |
| ufshpb_run_active_subregion_list(hpb); |
| } |
| |
| /* |
| * this function doesn't need to hold lock due to be called in init. |
| * (rgn_state_lock, rsp_list_lock, etc..) |
| */ |
| static int ufshpb_init_pinned_active_region(struct ufs_hba *hba, |
| struct ufshpb_lu *hpb, |
| struct ufshpb_region *rgn) |
| { |
| struct ufshpb_subregion *srgn; |
| int srgn_idx, i; |
| int err = 0; |
| |
| for_each_sub_region(rgn, srgn_idx, srgn) { |
| srgn->mctx = ufshpb_get_map_ctx(hpb, srgn->is_last); |
| srgn->srgn_state = HPB_SRGN_INVALID; |
| if (!srgn->mctx) { |
| err = -ENOMEM; |
| dev_err(hba->dev, |
| "alloc mctx for pinned region failed\n"); |
| goto release; |
| } |
| |
| list_add_tail(&srgn->list_act_srgn, &hpb->lh_act_srgn); |
| } |
| |
| rgn->rgn_state = HPB_RGN_PINNED; |
| return 0; |
| |
| release: |
| for (i = 0; i < srgn_idx; i++) { |
| srgn = rgn->srgn_tbl + i; |
| ufshpb_put_map_ctx(hpb, srgn->mctx); |
| } |
| return err; |
| } |
| |
| static void ufshpb_init_subregion_tbl(struct ufshpb_lu *hpb, |
| struct ufshpb_region *rgn, bool last) |
| { |
| int srgn_idx; |
| struct ufshpb_subregion *srgn; |
| |
| for_each_sub_region(rgn, srgn_idx, srgn) { |
| INIT_LIST_HEAD(&srgn->list_act_srgn); |
| |
| srgn->rgn_idx = rgn->rgn_idx; |
| srgn->srgn_idx = srgn_idx; |
| srgn->srgn_state = HPB_SRGN_UNUSED; |
| } |
| |
| if (unlikely(last && hpb->last_srgn_entries)) |
| srgn->is_last = true; |
| } |
| |
| static int ufshpb_alloc_subregion_tbl(struct ufshpb_lu *hpb, |
| struct ufshpb_region *rgn, int srgn_cnt) |
| { |
| rgn->srgn_tbl = kvcalloc(srgn_cnt, sizeof(struct ufshpb_subregion), |
| GFP_KERNEL); |
| if (!rgn->srgn_tbl) |
| return -ENOMEM; |
| |
| rgn->srgn_cnt = srgn_cnt; |
| return 0; |
| } |
| |
| static void ufshpb_lu_parameter_init(struct ufs_hba *hba, |
| struct ufshpb_lu *hpb, |
| struct ufshpb_dev_info *hpb_dev_info, |
| struct ufshpb_lu_info *hpb_lu_info) |
| { |
| u32 entries_per_rgn; |
| u64 rgn_mem_size, tmp; |
| |
| if (ufshpb_is_legacy(hba)) |
| hpb->pre_req_max_tr_len = HPB_LEGACY_CHUNK_HIGH; |
| else |
| hpb->pre_req_max_tr_len = hpb_dev_info->max_hpb_single_cmd; |
| |
| hpb->lu_pinned_start = hpb_lu_info->pinned_start; |
| hpb->lu_pinned_end = hpb_lu_info->num_pinned ? |
| (hpb_lu_info->pinned_start + hpb_lu_info->num_pinned - 1) |
| : PINNED_NOT_SET; |
| hpb->lru_info.max_lru_active_cnt = |
| hpb_lu_info->max_active_rgns - hpb_lu_info->num_pinned; |
| |
| rgn_mem_size = (1ULL << hpb_dev_info->rgn_size) * HPB_RGN_SIZE_UNIT |
| * HPB_ENTRY_SIZE; |
| do_div(rgn_mem_size, HPB_ENTRY_BLOCK_SIZE); |
| hpb->srgn_mem_size = (1ULL << hpb_dev_info->srgn_size) |
| * HPB_RGN_SIZE_UNIT / HPB_ENTRY_BLOCK_SIZE * HPB_ENTRY_SIZE; |
| |
| tmp = rgn_mem_size; |
| do_div(tmp, HPB_ENTRY_SIZE); |
| entries_per_rgn = (u32)tmp; |
| hpb->entries_per_rgn_shift = ilog2(entries_per_rgn); |
| hpb->entries_per_rgn_mask = entries_per_rgn - 1; |
| |
| hpb->entries_per_srgn = hpb->srgn_mem_size / HPB_ENTRY_SIZE; |
| hpb->entries_per_srgn_shift = ilog2(hpb->entries_per_srgn); |
| hpb->entries_per_srgn_mask = hpb->entries_per_srgn - 1; |
| |
| tmp = rgn_mem_size; |
| do_div(tmp, hpb->srgn_mem_size); |
| hpb->srgns_per_rgn = (int)tmp; |
| |
| hpb->rgns_per_lu = DIV_ROUND_UP(hpb_lu_info->num_blocks, |
| entries_per_rgn); |
| hpb->srgns_per_lu = DIV_ROUND_UP(hpb_lu_info->num_blocks, |
| (hpb->srgn_mem_size / HPB_ENTRY_SIZE)); |
| hpb->last_srgn_entries = hpb_lu_info->num_blocks |
| % (hpb->srgn_mem_size / HPB_ENTRY_SIZE); |
| |
| hpb->pages_per_srgn = DIV_ROUND_UP(hpb->srgn_mem_size, PAGE_SIZE); |
| |
| if (hpb_dev_info->control_mode == HPB_HOST_CONTROL) |
| hpb->is_hcm = true; |
| } |
| |
| static int ufshpb_alloc_region_tbl(struct ufs_hba *hba, struct ufshpb_lu *hpb) |
| { |
| struct ufshpb_region *rgn_table, *rgn; |
| int rgn_idx, i; |
| int ret = 0; |
| |
| rgn_table = kvcalloc(hpb->rgns_per_lu, sizeof(struct ufshpb_region), |
| GFP_KERNEL); |
| if (!rgn_table) |
| return -ENOMEM; |
| |
| for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) { |
| int srgn_cnt = hpb->srgns_per_rgn; |
| bool last_srgn = false; |
| |
| rgn = rgn_table + rgn_idx; |
| rgn->rgn_idx = rgn_idx; |
| |
| spin_lock_init(&rgn->rgn_lock); |
| |
| INIT_LIST_HEAD(&rgn->list_inact_rgn); |
| INIT_LIST_HEAD(&rgn->list_lru_rgn); |
| INIT_LIST_HEAD(&rgn->list_expired_rgn); |
| |
| if (rgn_idx == hpb->rgns_per_lu - 1) { |
| srgn_cnt = ((hpb->srgns_per_lu - 1) % |
| hpb->srgns_per_rgn) + 1; |
| last_srgn = true; |
| } |
| |
| ret = ufshpb_alloc_subregion_tbl(hpb, rgn, srgn_cnt); |
| if (ret) |
| goto release_srgn_table; |
| ufshpb_init_subregion_tbl(hpb, rgn, last_srgn); |
| |
| if (ufshpb_is_pinned_region(hpb, rgn_idx)) { |
| ret = ufshpb_init_pinned_active_region(hba, hpb, rgn); |
| if (ret) |
| goto release_srgn_table; |
| } else { |
| rgn->rgn_state = HPB_RGN_INACTIVE; |
| } |
| |
| rgn->rgn_flags = 0; |
| rgn->hpb = hpb; |
| } |
| |
| hpb->rgn_tbl = rgn_table; |
| |
| return 0; |
| |
| release_srgn_table: |
| for (i = 0; i <= rgn_idx; i++) |
| kvfree(rgn_table[i].srgn_tbl); |
| |
| kvfree(rgn_table); |
| return ret; |
| } |
| |
| static void ufshpb_destroy_subregion_tbl(struct ufshpb_lu *hpb, |
| struct ufshpb_region *rgn) |
| { |
| int srgn_idx; |
| struct ufshpb_subregion *srgn; |
| |
| for_each_sub_region(rgn, srgn_idx, srgn) |
| if (srgn->srgn_state != HPB_SRGN_UNUSED) { |
| srgn->srgn_state = HPB_SRGN_UNUSED; |
| ufshpb_put_map_ctx(hpb, srgn->mctx); |
| } |
| } |
| |
| static void ufshpb_destroy_region_tbl(struct ufshpb_lu *hpb) |
| { |
| int rgn_idx; |
| |
| for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) { |
| struct ufshpb_region *rgn; |
| |
| rgn = hpb->rgn_tbl + rgn_idx; |
| if (rgn->rgn_state != HPB_RGN_INACTIVE) { |
| rgn->rgn_state = HPB_RGN_INACTIVE; |
| |
| ufshpb_destroy_subregion_tbl(hpb, rgn); |
| } |
| |
| kvfree(rgn->srgn_tbl); |
| } |
| |
| kvfree(hpb->rgn_tbl); |
| } |
| |
| /* SYSFS functions */ |
| #define ufshpb_sysfs_attr_show_func(__name) \ |
| static ssize_t __name##_show(struct device *dev, \ |
| struct device_attribute *attr, char *buf) \ |
| { \ |
| struct scsi_device *sdev = to_scsi_device(dev); \ |
| struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); \ |
| \ |
| if (!hpb) \ |
| return -ENODEV; \ |
| \ |
| return sysfs_emit(buf, "%llu\n", hpb->stats.__name); \ |
| } \ |
| \ |
| static DEVICE_ATTR_RO(__name) |
| |
| ufshpb_sysfs_attr_show_func(hit_cnt); |
| ufshpb_sysfs_attr_show_func(miss_cnt); |
| ufshpb_sysfs_attr_show_func(rcmd_noti_cnt); |
| ufshpb_sysfs_attr_show_func(rcmd_active_cnt); |
| ufshpb_sysfs_attr_show_func(rcmd_inactive_cnt); |
| ufshpb_sysfs_attr_show_func(map_req_cnt); |
| ufshpb_sysfs_attr_show_func(umap_req_cnt); |
| |
| static struct attribute *hpb_dev_stat_attrs[] = { |
| &dev_attr_hit_cnt.attr, |
| &dev_attr_miss_cnt.attr, |
| &dev_attr_rcmd_noti_cnt.attr, |
| &dev_attr_rcmd_active_cnt.attr, |
| &dev_attr_rcmd_inactive_cnt.attr, |
| &dev_attr_map_req_cnt.attr, |
| &dev_attr_umap_req_cnt.attr, |
| NULL, |
| }; |
| |
| struct attribute_group ufs_sysfs_hpb_stat_group = { |
| .name = "hpb_stats", |
| .attrs = hpb_dev_stat_attrs, |
| }; |
| |
| /* SYSFS functions */ |
| #define ufshpb_sysfs_param_show_func(__name) \ |
| static ssize_t __name##_show(struct device *dev, \ |
| struct device_attribute *attr, char *buf) \ |
| { \ |
| struct scsi_device *sdev = to_scsi_device(dev); \ |
| struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); \ |
| \ |
| if (!hpb) \ |
| return -ENODEV; \ |
| \ |
| return sysfs_emit(buf, "%d\n", hpb->params.__name); \ |
| } |
| |
| ufshpb_sysfs_param_show_func(requeue_timeout_ms); |
| static ssize_t |
| requeue_timeout_ms_store(struct device *dev, struct device_attribute *attr, |
| const char *buf, size_t count) |
| { |
| struct scsi_device *sdev = to_scsi_device(dev); |
| struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); |
| int val; |
| |
| if (!hpb) |
| return -ENODEV; |
| |
| if (kstrtouint(buf, 0, &val)) |
| return -EINVAL; |
| |
| if (val < 0) |
| return -EINVAL; |
| |
| hpb->params.requeue_timeout_ms = val; |
| |
| return count; |
| } |
| static DEVICE_ATTR_RW(requeue_timeout_ms); |
| |
| ufshpb_sysfs_param_show_func(activation_thld); |
| static ssize_t |
| activation_thld_store(struct device *dev, struct device_attribute *attr, |
| const char *buf, size_t count) |
| { |
| struct scsi_device *sdev = to_scsi_device(dev); |
| struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); |
| int val; |
| |
| if (!hpb) |
| return -ENODEV; |
| |
| if (!hpb->is_hcm) |
| return -EOPNOTSUPP; |
| |
| if (kstrtouint(buf, 0, &val)) |
| return -EINVAL; |
| |
| if (val <= 0) |
| return -EINVAL; |
| |
| hpb->params.activation_thld = val; |
| |
| return count; |
| } |
| static DEVICE_ATTR_RW(activation_thld); |
| |
| ufshpb_sysfs_param_show_func(normalization_factor); |
| static ssize_t |
| normalization_factor_store(struct device *dev, struct device_attribute *attr, |
| const char *buf, size_t count) |
| { |
| struct scsi_device *sdev = to_scsi_device(dev); |
| struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); |
| int val; |
| |
| if (!hpb) |
| return -ENODEV; |
| |
| if (!hpb->is_hcm) |
| return -EOPNOTSUPP; |
| |
| if (kstrtouint(buf, 0, &val)) |
| return -EINVAL; |
| |
| if (val <= 0 || val > ilog2(hpb->entries_per_srgn)) |
| return -EINVAL; |
| |
| hpb->params.normalization_factor = val; |
| |
| return count; |
| } |
| static DEVICE_ATTR_RW(normalization_factor); |
| |
| ufshpb_sysfs_param_show_func(eviction_thld_enter); |
| static ssize_t |
| eviction_thld_enter_store(struct device *dev, struct device_attribute *attr, |
| const char *buf, size_t count) |
| { |
| struct scsi_device *sdev = to_scsi_device(dev); |
| struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); |
| int val; |
| |
| if (!hpb) |
| return -ENODEV; |
| |
| if (!hpb->is_hcm) |
| return -EOPNOTSUPP; |
| |
| if (kstrtouint(buf, 0, &val)) |
| return -EINVAL; |
| |
| if (val <= hpb->params.eviction_thld_exit) |
| return -EINVAL; |
| |
| hpb->params.eviction_thld_enter = val; |
| |
| return count; |
| } |
| static DEVICE_ATTR_RW(eviction_thld_enter); |
| |
| ufshpb_sysfs_param_show_func(eviction_thld_exit); |
| static ssize_t |
| eviction_thld_exit_store(struct device *dev, struct device_attribute *attr, |
| const char *buf, size_t count) |
| { |
| struct scsi_device *sdev = to_scsi_device(dev); |
| struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); |
| int val; |
| |
| if (!hpb) |
| return -ENODEV; |
| |
| if (!hpb->is_hcm) |
| return -EOPNOTSUPP; |
| |
| if (kstrtouint(buf, 0, &val)) |
| return -EINVAL; |
| |
| if (val <= hpb->params.activation_thld) |
| return -EINVAL; |
| |
| hpb->params.eviction_thld_exit = val; |
| |
| return count; |
| } |
| static DEVICE_ATTR_RW(eviction_thld_exit); |
| |
| ufshpb_sysfs_param_show_func(read_timeout_ms); |
| static ssize_t |
| read_timeout_ms_store(struct device *dev, struct device_attribute *attr, |
| const char *buf, size_t count) |
| { |
| struct scsi_device *sdev = to_scsi_device(dev); |
| struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); |
| int val; |
| |
| if (!hpb) |
| return -ENODEV; |
| |
| if (!hpb->is_hcm) |
| return -EOPNOTSUPP; |
| |
| if (kstrtouint(buf, 0, &val)) |
| return -EINVAL; |
| |
| /* read_timeout >> timeout_polling_interval */ |
| if (val < hpb->params.timeout_polling_interval_ms * 2) |
| return -EINVAL; |
| |
| hpb->params.read_timeout_ms = val; |
| |
| return count; |
| } |
| static DEVICE_ATTR_RW(read_timeout_ms); |
| |
| ufshpb_sysfs_param_show_func(read_timeout_expiries); |
| static ssize_t |
| read_timeout_expiries_store(struct device *dev, struct device_attribute *attr, |
| const char *buf, size_t count) |
| { |
| struct scsi_device *sdev = to_scsi_device(dev); |
| struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); |
| int val; |
| |
| if (!hpb) |
| return -ENODEV; |
| |
| if (!hpb->is_hcm) |
| return -EOPNOTSUPP; |
| |
| if (kstrtouint(buf, 0, &val)) |
| return -EINVAL; |
| |
| if (val <= 0) |
| return -EINVAL; |
| |
| hpb->params.read_timeout_expiries = val; |
| |
| return count; |
| } |
| static DEVICE_ATTR_RW(read_timeout_expiries); |
| |
| ufshpb_sysfs_param_show_func(timeout_polling_interval_ms); |
| static ssize_t |
| timeout_polling_interval_ms_store(struct device *dev, |
| struct device_attribute *attr, |
| const char *buf, size_t count) |
| { |
| struct scsi_device *sdev = to_scsi_device(dev); |
| struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); |
| int val; |
| |
| if (!hpb) |
| return -ENODEV; |
| |
| if (!hpb->is_hcm) |
| return -EOPNOTSUPP; |
| |
| if (kstrtouint(buf, 0, &val)) |
| return -EINVAL; |
| |
| /* timeout_polling_interval << read_timeout */ |
| if (val <= 0 || val > hpb->params.read_timeout_ms / 2) |
| return -EINVAL; |
| |
| hpb->params.timeout_polling_interval_ms = val; |
| |
| return count; |
| } |
| static DEVICE_ATTR_RW(timeout_polling_interval_ms); |
| |
| ufshpb_sysfs_param_show_func(inflight_map_req); |
| static ssize_t inflight_map_req_store(struct device *dev, |
| struct device_attribute *attr, |
| const char *buf, size_t count) |
| { |
| struct scsi_device *sdev = to_scsi_device(dev); |
| struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); |
| int val; |
| |
| if (!hpb) |
| return -ENODEV; |
| |
| if (!hpb->is_hcm) |
| return -EOPNOTSUPP; |
| |
| if (kstrtouint(buf, 0, &val)) |
| return -EINVAL; |
| |
| if (val <= 0 || val > hpb->sdev_ufs_lu->queue_depth - 1) |
| return -EINVAL; |
| |
| hpb->params.inflight_map_req = val; |
| |
| return count; |
| } |
| static DEVICE_ATTR_RW(inflight_map_req); |
| |
| static void ufshpb_hcm_param_init(struct ufshpb_lu *hpb) |
| { |
| hpb->params.activation_thld = ACTIVATION_THRESHOLD; |
| hpb->params.normalization_factor = 1; |
| hpb->params.eviction_thld_enter = (ACTIVATION_THRESHOLD << 5); |
| hpb->params.eviction_thld_exit = (ACTIVATION_THRESHOLD << 4); |
| hpb->params.read_timeout_ms = READ_TO_MS; |
| hpb->params.read_timeout_expiries = READ_TO_EXPIRIES; |
| hpb->params.timeout_polling_interval_ms = POLLING_INTERVAL_MS; |
| hpb->params.inflight_map_req = THROTTLE_MAP_REQ_DEFAULT; |
| } |
| |
| static struct attribute *hpb_dev_param_attrs[] = { |
| &dev_attr_requeue_timeout_ms.attr, |
| &dev_attr_activation_thld.attr, |
| &dev_attr_normalization_factor.attr, |
| &dev_attr_eviction_thld_enter.attr, |
| &dev_attr_eviction_thld_exit.attr, |
| &dev_attr_read_timeout_ms.attr, |
| &dev_attr_read_timeout_expiries.attr, |
| &dev_attr_timeout_polling_interval_ms.attr, |
| &dev_attr_inflight_map_req.attr, |
| NULL, |
| }; |
| |
| struct attribute_group ufs_sysfs_hpb_param_group = { |
| .name = "hpb_params", |
| .attrs = hpb_dev_param_attrs, |
| }; |
| |
| static int ufshpb_pre_req_mempool_init(struct ufshpb_lu *hpb) |
| { |
| struct ufshpb_req *pre_req = NULL, *t; |
| int qd = hpb->sdev_ufs_lu->queue_depth / 2; |
| int i; |
| |
| INIT_LIST_HEAD(&hpb->lh_pre_req_free); |
| |
| hpb->pre_req = kcalloc(qd, sizeof(struct ufshpb_req), GFP_KERNEL); |
| hpb->throttle_pre_req = qd; |
| hpb->num_inflight_pre_req = 0; |
| |
| if (!hpb->pre_req) |
| goto release_mem; |
| |
| for (i = 0; i < qd; i++) { |
| pre_req = hpb->pre_req + i; |
| INIT_LIST_HEAD(&pre_req->list_req); |
| pre_req->req = NULL; |
| |
| pre_req->bio = bio_alloc(NULL, 1, 0, GFP_KERNEL); |
| if (!pre_req->bio) |
| goto release_mem; |
| |
| pre_req->wb.m_page = alloc_page(GFP_KERNEL | __GFP_ZERO); |
| if (!pre_req->wb.m_page) { |
| bio_put(pre_req->bio); |
| goto release_mem; |
| } |
| |
| list_add_tail(&pre_req->list_req, &hpb->lh_pre_req_free); |
| } |
| |
| return 0; |
| release_mem: |
| list_for_each_entry_safe(pre_req, t, &hpb->lh_pre_req_free, list_req) { |
| list_del_init(&pre_req->list_req); |
| bio_put(pre_req->bio); |
| __free_page(pre_req->wb.m_page); |
| } |
| |
| kfree(hpb->pre_req); |
| return -ENOMEM; |
| } |
| |
| static void ufshpb_pre_req_mempool_destroy(struct ufshpb_lu *hpb) |
| { |
| struct ufshpb_req *pre_req = NULL; |
| int i; |
| |
| for (i = 0; i < hpb->throttle_pre_req; i++) { |
| pre_req = hpb->pre_req + i; |
| bio_put(hpb->pre_req[i].bio); |
| if (!pre_req->wb.m_page) |
| __free_page(hpb->pre_req[i].wb.m_page); |
| list_del_init(&pre_req->list_req); |
| } |
| |
| kfree(hpb->pre_req); |
| } |
| |
| static void ufshpb_stat_init(struct ufshpb_lu *hpb) |
| { |
| hpb->stats.hit_cnt = 0; |
| hpb->stats.miss_cnt = 0; |
| hpb->stats.rcmd_noti_cnt = 0; |
| hpb->stats.rcmd_active_cnt = 0; |
| hpb->stats.rcmd_inactive_cnt = 0; |
| hpb->stats.map_req_cnt = 0; |
| hpb->stats.umap_req_cnt = 0; |
| } |
| |
| static void ufshpb_param_init(struct ufshpb_lu *hpb) |
| { |
| hpb->params.requeue_timeout_ms = HPB_REQUEUE_TIME_MS; |
| if (hpb->is_hcm) |
| ufshpb_hcm_param_init(hpb); |
| } |
| |
| static int ufshpb_lu_hpb_init(struct ufs_hba *hba, struct ufshpb_lu *hpb) |
| { |
| int ret; |
| |
| spin_lock_init(&hpb->rgn_state_lock); |
| spin_lock_init(&hpb->rsp_list_lock); |
| spin_lock_init(&hpb->param_lock); |
| |
| INIT_LIST_HEAD(&hpb->lru_info.lh_lru_rgn); |
| INIT_LIST_HEAD(&hpb->lh_act_srgn); |
| INIT_LIST_HEAD(&hpb->lh_inact_rgn); |
| INIT_LIST_HEAD(&hpb->list_hpb_lu); |
| |
| INIT_WORK(&hpb->map_work, ufshpb_map_work_handler); |
| if (hpb->is_hcm) { |
| INIT_WORK(&hpb->ufshpb_normalization_work, |
| ufshpb_normalization_work_handler); |
| INIT_DELAYED_WORK(&hpb->ufshpb_read_to_work, |
| ufshpb_read_to_handler); |
| } |
| |
| hpb->map_req_cache = kmem_cache_create("ufshpb_req_cache", |
| sizeof(struct ufshpb_req), 0, 0, NULL); |
| if (!hpb->map_req_cache) { |
| dev_err(hba->dev, "ufshpb(%d) ufshpb_req_cache create fail", |
| hpb->lun); |
| return -ENOMEM; |
| } |
| |
| hpb->m_page_cache = kmem_cache_create("ufshpb_m_page_cache", |
| sizeof(struct page *) * hpb->pages_per_srgn, |
| 0, 0, NULL); |
| if (!hpb->m_page_cache) { |
| dev_err(hba->dev, "ufshpb(%d) ufshpb_m_page_cache create fail", |
| hpb->lun); |
| ret = -ENOMEM; |
| goto release_req_cache; |
| } |
| |
| ret = ufshpb_pre_req_mempool_init(hpb); |
| if (ret) { |
| dev_err(hba->dev, "ufshpb(%d) pre_req_mempool init fail", |
| hpb->lun); |
| goto release_m_page_cache; |
| } |
| |
| ret = ufshpb_alloc_region_tbl(hba, hpb); |
| if (ret) |
| goto release_pre_req_mempool; |
| |
| ufshpb_stat_init(hpb); |
| ufshpb_param_init(hpb); |
| |
| if (hpb->is_hcm) { |
| unsigned int poll; |
| |
| poll = hpb->params.timeout_polling_interval_ms; |
| schedule_delayed_work(&hpb->ufshpb_read_to_work, |
| msecs_to_jiffies(poll)); |
| } |
| |
| return 0; |
| |
| release_pre_req_mempool: |
| ufshpb_pre_req_mempool_destroy(hpb); |
| release_m_page_cache: |
| kmem_cache_destroy(hpb->m_page_cache); |
| release_req_cache: |
| kmem_cache_destroy(hpb->map_req_cache); |
| return ret; |
| } |
| |
| static struct ufshpb_lu * |
| ufshpb_alloc_hpb_lu(struct ufs_hba *hba, struct scsi_device *sdev, |
| struct ufshpb_dev_info *hpb_dev_info, |
| struct ufshpb_lu_info *hpb_lu_info) |
| { |
| struct ufshpb_lu *hpb; |
| int ret; |
| |
| hpb = kzalloc(sizeof(struct ufshpb_lu), GFP_KERNEL); |
| if (!hpb) |
| return NULL; |
| |
| hpb->lun = sdev->lun; |
| hpb->sdev_ufs_lu = sdev; |
| |
| ufshpb_lu_parameter_init(hba, hpb, hpb_dev_info, hpb_lu_info); |
| |
| ret = ufshpb_lu_hpb_init(hba, hpb); |
| if (ret) { |
| dev_err(hba->dev, "hpb lu init failed. ret %d", ret); |
| goto release_hpb; |
| } |
| |
| sdev->hostdata = hpb; |
| return hpb; |
| |
| release_hpb: |
| kfree(hpb); |
| return NULL; |
| } |
| |
| static void ufshpb_discard_rsp_lists(struct ufshpb_lu *hpb) |
| { |
| struct ufshpb_region *rgn, *next_rgn; |
| struct ufshpb_subregion *srgn, *next_srgn; |
| unsigned long flags; |
| |
| /* |
| * If the device reset occurred, the remaining HPB region information |
| * may be stale. Therefore, by discarding the lists of HPB response |
| * that remained after reset, we prevent unnecessary work. |
| */ |
| spin_lock_irqsave(&hpb->rsp_list_lock, flags); |
| list_for_each_entry_safe(rgn, next_rgn, &hpb->lh_inact_rgn, |
| list_inact_rgn) |
| list_del_init(&rgn->list_inact_rgn); |
| |
| list_for_each_entry_safe(srgn, next_srgn, &hpb->lh_act_srgn, |
| list_act_srgn) |
| list_del_init(&srgn->list_act_srgn); |
| spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); |
| } |
| |
| static void ufshpb_cancel_jobs(struct ufshpb_lu *hpb) |
| { |
| if (hpb->is_hcm) { |
| cancel_delayed_work_sync(&hpb->ufshpb_read_to_work); |
| cancel_work_sync(&hpb->ufshpb_normalization_work); |
| } |
| cancel_work_sync(&hpb->map_work); |
| } |
| |
| static bool ufshpb_check_hpb_reset_query(struct ufs_hba *hba) |
| { |
| int err = 0; |
| bool flag_res = true; |
| int try; |
| |
| /* wait for the device to complete HPB reset query */ |
| for (try = 0; try < HPB_RESET_REQ_RETRIES; try++) { |
| dev_dbg(hba->dev, |
| "%s start flag reset polling %d times\n", |
| __func__, try); |
| |
| /* Poll fHpbReset flag to be cleared */ |
| err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG, |
| QUERY_FLAG_IDN_HPB_RESET, 0, &flag_res); |
| |
| if (err) { |
| dev_err(hba->dev, |
| "%s reading fHpbReset flag failed with error %d\n", |
| __func__, err); |
| return flag_res; |
| } |
| |
| if (!flag_res) |
| goto out; |
| |
| usleep_range(1000, 1100); |
| } |
| if (flag_res) { |
| dev_err(hba->dev, |
| "%s fHpbReset was not cleared by the device\n", |
| __func__); |
| } |
| out: |
| return flag_res; |
| } |
| |
| /** |
| * ufshpb_toggle_state - switch HPB state of all LUs |
| * @hba: per-adapter instance |
| * @src: expected current HPB state |
| * @dest: target HPB state to switch to |
| */ |
| void ufshpb_toggle_state(struct ufs_hba *hba, enum UFSHPB_STATE src, enum UFSHPB_STATE dest) |
| { |
| struct ufshpb_lu *hpb; |
| struct scsi_device *sdev; |
| |
| shost_for_each_device(sdev, hba->host) { |
| hpb = ufshpb_get_hpb_data(sdev); |
| |
| if (!hpb || ufshpb_get_state(hpb) != src) |
| continue; |
| ufshpb_set_state(hpb, dest); |
| |
| if (dest == HPB_RESET) { |
| ufshpb_cancel_jobs(hpb); |
| ufshpb_discard_rsp_lists(hpb); |
| } |
| } |
| } |
| |
| void ufshpb_suspend(struct ufs_hba *hba) |
| { |
| struct ufshpb_lu *hpb; |
| struct scsi_device *sdev; |
| |
| shost_for_each_device(sdev, hba->host) { |
| hpb = ufshpb_get_hpb_data(sdev); |
| if (!hpb || ufshpb_get_state(hpb) != HPB_PRESENT) |
| continue; |
| |
| ufshpb_set_state(hpb, HPB_SUSPEND); |
| ufshpb_cancel_jobs(hpb); |
| } |
| } |
| |
| void ufshpb_resume(struct ufs_hba *hba) |
| { |
| struct ufshpb_lu *hpb; |
| struct scsi_device *sdev; |
| |
| shost_for_each_device(sdev, hba->host) { |
| hpb = ufshpb_get_hpb_data(sdev); |
| if (!hpb || ufshpb_get_state(hpb) != HPB_SUSPEND) |
| continue; |
| |
| ufshpb_set_state(hpb, HPB_PRESENT); |
| ufshpb_kick_map_work(hpb); |
| if (hpb->is_hcm) { |
| unsigned int poll = hpb->params.timeout_polling_interval_ms; |
| |
| schedule_delayed_work(&hpb->ufshpb_read_to_work, msecs_to_jiffies(poll)); |
| } |
| } |
| } |
| |
| static int ufshpb_get_lu_info(struct ufs_hba *hba, int lun, |
| struct ufshpb_lu_info *hpb_lu_info) |
| { |
| u16 max_active_rgns; |
| u8 lu_enable; |
| int size; |
| int ret; |
| char desc_buf[QUERY_DESC_MAX_SIZE]; |
| |
| ufshcd_map_desc_id_to_length(hba, QUERY_DESC_IDN_UNIT, &size); |
| |
| ufshcd_rpm_get_sync(hba); |
| ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, |
| QUERY_DESC_IDN_UNIT, lun, 0, |
| desc_buf, &size); |
| ufshcd_rpm_put_sync(hba); |
| |
| if (ret) { |
| dev_err(hba->dev, |
| "%s: idn: %d lun: %d query request failed", |
| __func__, QUERY_DESC_IDN_UNIT, lun); |
| return ret; |
| } |
| |
| lu_enable = desc_buf[UNIT_DESC_PARAM_LU_ENABLE]; |
| if (lu_enable != LU_ENABLED_HPB_FUNC) |
| return -ENODEV; |
| |
| max_active_rgns = get_unaligned_be16( |
| desc_buf + UNIT_DESC_PARAM_HPB_LU_MAX_ACTIVE_RGNS); |
| if (!max_active_rgns) { |
| dev_err(hba->dev, |
| "lun %d wrong number of max active regions\n", lun); |
| return -ENODEV; |
| } |
| |
| hpb_lu_info->num_blocks = get_unaligned_be64( |
| desc_buf + UNIT_DESC_PARAM_LOGICAL_BLK_COUNT); |
| hpb_lu_info->pinned_start = get_unaligned_be16( |
| desc_buf + UNIT_DESC_PARAM_HPB_PIN_RGN_START_OFF); |
| hpb_lu_info->num_pinned = get_unaligned_be16( |
| desc_buf + UNIT_DESC_PARAM_HPB_NUM_PIN_RGNS); |
| hpb_lu_info->max_active_rgns = max_active_rgns; |
| |
| return 0; |
| } |
| |
| void ufshpb_destroy_lu(struct ufs_hba *hba, struct scsi_device *sdev) |
| { |
| struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); |
| |
| if (!hpb) |
| return; |
| |
| ufshpb_set_state(hpb, HPB_FAILED); |
| |
| sdev = hpb->sdev_ufs_lu; |
| sdev->hostdata = NULL; |
| |
| ufshpb_cancel_jobs(hpb); |
| |
| ufshpb_pre_req_mempool_destroy(hpb); |
| ufshpb_destroy_region_tbl(hpb); |
| |
| kmem_cache_destroy(hpb->map_req_cache); |
| kmem_cache_destroy(hpb->m_page_cache); |
| |
| list_del_init(&hpb->list_hpb_lu); |
| |
| kfree(hpb); |
| } |
| |
| static void ufshpb_hpb_lu_prepared(struct ufs_hba *hba) |
| { |
| int pool_size; |
| struct ufshpb_lu *hpb; |
| struct scsi_device *sdev; |
| bool init_success; |
| |
| if (tot_active_srgn_pages == 0) { |
| ufshpb_remove(hba); |
| return; |
| } |
| |
| init_success = !ufshpb_check_hpb_reset_query(hba); |
| |
| pool_size = PAGE_ALIGN(ufshpb_host_map_kbytes * 1024) / PAGE_SIZE; |
| if (pool_size > tot_active_srgn_pages) { |
| mempool_resize(ufshpb_mctx_pool, tot_active_srgn_pages); |
| mempool_resize(ufshpb_page_pool, tot_active_srgn_pages); |
| } |
| |
| shost_for_each_device(sdev, hba->host) { |
| hpb = ufshpb_get_hpb_data(sdev); |
| if (!hpb) |
| continue; |
| |
| if (init_success) { |
| ufshpb_set_state(hpb, HPB_PRESENT); |
| if ((hpb->lu_pinned_end - hpb->lu_pinned_start) > 0) |
| queue_work(ufshpb_wq, &hpb->map_work); |
| } else { |
| dev_err(hba->dev, "destroy HPB lu %d\n", hpb->lun); |
| ufshpb_destroy_lu(hba, sdev); |
| } |
| } |
| |
| if (!init_success) |
| ufshpb_remove(hba); |
| } |
| |
| void ufshpb_init_hpb_lu(struct ufs_hba *hba, struct scsi_device *sdev) |
| { |
| struct ufshpb_lu *hpb; |
| int ret; |
| struct ufshpb_lu_info hpb_lu_info = { 0 }; |
| int lun = sdev->lun; |
| |
| if (lun >= hba->dev_info.max_lu_supported) |
| goto out; |
| |
| ret = ufshpb_get_lu_info(hba, lun, &hpb_lu_info); |
| if (ret) |
| goto out; |
| |
| hpb = ufshpb_alloc_hpb_lu(hba, sdev, &hba->ufshpb_dev, |
| &hpb_lu_info); |
| if (!hpb) |
| goto out; |
| |
| tot_active_srgn_pages += hpb_lu_info.max_active_rgns * |
| hpb->srgns_per_rgn * hpb->pages_per_srgn; |
| |
| out: |
| /* All LUs are initialized */ |
| if (atomic_dec_and_test(&hba->ufshpb_dev.slave_conf_cnt)) |
| ufshpb_hpb_lu_prepared(hba); |
| } |
| |
| static int ufshpb_init_mem_wq(struct ufs_hba *hba) |
| { |
| int ret; |
| unsigned int pool_size; |
| |
| ufshpb_mctx_cache = kmem_cache_create("ufshpb_mctx_cache", |
| sizeof(struct ufshpb_map_ctx), |
| 0, 0, NULL); |
| if (!ufshpb_mctx_cache) { |
| dev_err(hba->dev, "ufshpb: cannot init mctx cache\n"); |
| return -ENOMEM; |
| } |
| |
| pool_size = PAGE_ALIGN(ufshpb_host_map_kbytes * 1024) / PAGE_SIZE; |
| dev_info(hba->dev, "%s:%d ufshpb_host_map_kbytes %u pool_size %u\n", |
| __func__, __LINE__, ufshpb_host_map_kbytes, pool_size); |
| |
| ufshpb_mctx_pool = mempool_create_slab_pool(pool_size, |
| ufshpb_mctx_cache); |
| if (!ufshpb_mctx_pool) { |
| dev_err(hba->dev, "ufshpb: cannot init mctx pool\n"); |
| ret = -ENOMEM; |
| goto release_mctx_cache; |
| } |
| |
| ufshpb_page_pool = mempool_create_page_pool(pool_size, 0); |
| if (!ufshpb_page_pool) { |
| dev_err(hba->dev, "ufshpb: cannot init page pool\n"); |
| ret = -ENOMEM; |
| goto release_mctx_pool; |
| } |
| |
| ufshpb_wq = alloc_workqueue("ufshpb-wq", |
| WQ_UNBOUND | WQ_MEM_RECLAIM, 0); |
| if (!ufshpb_wq) { |
| dev_err(hba->dev, "ufshpb: alloc workqueue failed\n"); |
| ret = -ENOMEM; |
| goto release_page_pool; |
| } |
| |
| return 0; |
| |
| release_page_pool: |
| mempool_destroy(ufshpb_page_pool); |
| release_mctx_pool: |
| mempool_destroy(ufshpb_mctx_pool); |
| release_mctx_cache: |
| kmem_cache_destroy(ufshpb_mctx_cache); |
| return ret; |
| } |
| |
| void ufshpb_get_geo_info(struct ufs_hba *hba, u8 *geo_buf) |
| { |
| struct ufshpb_dev_info *hpb_info = &hba->ufshpb_dev; |
| int max_active_rgns = 0; |
| int hpb_num_lu; |
| |
| hpb_num_lu = geo_buf[GEOMETRY_DESC_PARAM_HPB_NUMBER_LU]; |
| if (hpb_num_lu == 0) { |
| dev_err(hba->dev, "No HPB LU supported\n"); |
| hpb_info->hpb_disabled = true; |
| return; |
| } |
| |
| hpb_info->rgn_size = geo_buf[GEOMETRY_DESC_PARAM_HPB_REGION_SIZE]; |
| hpb_info->srgn_size = geo_buf[GEOMETRY_DESC_PARAM_HPB_SUBREGION_SIZE]; |
| max_active_rgns = get_unaligned_be16(geo_buf + |
| GEOMETRY_DESC_PARAM_HPB_MAX_ACTIVE_REGS); |
| |
| if (hpb_info->rgn_size == 0 || hpb_info->srgn_size == 0 || |
| max_active_rgns == 0) { |
| dev_err(hba->dev, "No HPB supported device\n"); |
| hpb_info->hpb_disabled = true; |
| return; |
| } |
| } |
| |
| void ufshpb_get_dev_info(struct ufs_hba *hba, u8 *desc_buf) |
| { |
| struct ufshpb_dev_info *hpb_dev_info = &hba->ufshpb_dev; |
| int version, ret; |
| int max_single_cmd; |
| |
| hpb_dev_info->control_mode = desc_buf[DEVICE_DESC_PARAM_HPB_CONTROL]; |
| |
| version = get_unaligned_be16(desc_buf + DEVICE_DESC_PARAM_HPB_VER); |
| if ((version != HPB_SUPPORT_VERSION) && |
| (version != HPB_SUPPORT_LEGACY_VERSION)) { |
| dev_err(hba->dev, "%s: HPB %x version is not supported.\n", |
| __func__, version); |
| hpb_dev_info->hpb_disabled = true; |
| return; |
| } |
| |
| if (version == HPB_SUPPORT_LEGACY_VERSION) |
| hpb_dev_info->is_legacy = true; |
| |
| /* |
| * Get the number of user logical unit to check whether all |
| * scsi_device finish initialization |
| */ |
| hpb_dev_info->num_lu = desc_buf[DEVICE_DESC_PARAM_NUM_LU]; |
| |
| if (hpb_dev_info->is_legacy) |
| return; |
| |
| ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, |
| QUERY_ATTR_IDN_MAX_HPB_SINGLE_CMD, 0, 0, &max_single_cmd); |
| |
| if (ret) |
| hpb_dev_info->max_hpb_single_cmd = HPB_LEGACY_CHUNK_HIGH; |
| else |
| hpb_dev_info->max_hpb_single_cmd = min(max_single_cmd + 1, HPB_MULTI_CHUNK_HIGH); |
| } |
| |
| void ufshpb_init(struct ufs_hba *hba) |
| { |
| struct ufshpb_dev_info *hpb_dev_info = &hba->ufshpb_dev; |
| int try; |
| int ret; |
| |
| if (!ufshpb_is_allowed(hba) || !hba->dev_info.hpb_enabled) |
| return; |
| |
| if (ufshpb_init_mem_wq(hba)) { |
| hpb_dev_info->hpb_disabled = true; |
| return; |
| } |
| |
| atomic_set(&hpb_dev_info->slave_conf_cnt, hpb_dev_info->num_lu); |
| tot_active_srgn_pages = 0; |
| /* issue HPB reset query */ |
| for (try = 0; try < HPB_RESET_REQ_RETRIES; try++) { |
| ret = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG, |
| QUERY_FLAG_IDN_HPB_RESET, 0, NULL); |
| if (!ret) |
| break; |
| } |
| } |
| |
| void ufshpb_remove(struct ufs_hba *hba) |
| { |
| mempool_destroy(ufshpb_page_pool); |
| mempool_destroy(ufshpb_mctx_pool); |
| kmem_cache_destroy(ufshpb_mctx_cache); |
| |
| destroy_workqueue(ufshpb_wq); |
| } |
| |
| module_param(ufshpb_host_map_kbytes, uint, 0644); |
| MODULE_PARM_DESC(ufshpb_host_map_kbytes, |
| "ufshpb host mapping memory kilo-bytes for ufshpb memory-pool"); |