| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Daejun Park | 9038 | 76.15% | 4 | 17.39% |
| Avri Altman | 2803 | 23.62% | 15 | 65.22% |
| Martin K. Petersen | 18 | 0.15% | 1 | 4.35% |
| Keoseong Park | 8 | 0.07% | 2 | 8.70% |
| Bean Huo | 2 | 0.02% | 1 | 4.35% |
| Total | 11869 | 23 |
// SPDX-License-Identifier: GPL-2.0 /* * Universal Flash Storage Host Performance Booster * * Copyright (C) 2017-2021 Samsung Electronics Co., Ltd. * * Authors: * Yongmyung Lee <ymhungry.lee@samsung.com> * Jinyoung Choi <j-young.choi@samsung.com> */ #include <asm/unaligned.h> #include <linux/async.h> #include "ufshcd.h" #include "ufshpb.h" #include "../sd.h" #define ACTIVATION_THRESHOLD 8 /* 8 IOs */ #define READ_TO_MS 1000 #define READ_TO_EXPIRIES 100 #define POLLING_INTERVAL_MS 200 #define THROTTLE_MAP_REQ_DEFAULT 1 /* memory management */ static struct kmem_cache *ufshpb_mctx_cache; static mempool_t *ufshpb_mctx_pool; static mempool_t *ufshpb_page_pool; /* A cache size of 2MB can cache ppn in the 1GB range. */ static unsigned int ufshpb_host_map_kbytes = 2048; static int tot_active_srgn_pages; static struct workqueue_struct *ufshpb_wq; static void ufshpb_update_active_info(struct ufshpb_lu *hpb, int rgn_idx, int srgn_idx); bool ufshpb_is_allowed(struct ufs_hba *hba) { return !(hba->ufshpb_dev.hpb_disabled); } /* HPB version 1.0 is called as legacy version. */ bool ufshpb_is_legacy(struct ufs_hba *hba) { return hba->ufshpb_dev.is_legacy; } static struct ufshpb_lu *ufshpb_get_hpb_data(struct scsi_device *sdev) { return sdev->hostdata; } static int ufshpb_get_state(struct ufshpb_lu *hpb) { return atomic_read(&hpb->hpb_state); } static void ufshpb_set_state(struct ufshpb_lu *hpb, int state) { atomic_set(&hpb->hpb_state, state); } static int ufshpb_is_valid_srgn(struct ufshpb_region *rgn, struct ufshpb_subregion *srgn) { return rgn->rgn_state != HPB_RGN_INACTIVE && srgn->srgn_state == HPB_SRGN_VALID; } static bool ufshpb_is_read_cmd(struct scsi_cmnd *cmd) { return req_op(scsi_cmd_to_rq(cmd)) == REQ_OP_READ; } static bool ufshpb_is_write_or_discard(struct scsi_cmnd *cmd) { return op_is_write(req_op(scsi_cmd_to_rq(cmd))) || op_is_discard(req_op(scsi_cmd_to_rq(cmd))); } static bool ufshpb_is_supported_chunk(struct ufshpb_lu *hpb, int transfer_len) { return transfer_len <= hpb->pre_req_max_tr_len; } static bool ufshpb_is_general_lun(int lun) { return lun < UFS_UPIU_MAX_UNIT_NUM_ID; } static bool ufshpb_is_pinned_region(struct ufshpb_lu *hpb, int rgn_idx) { if (hpb->lu_pinned_end != PINNED_NOT_SET && rgn_idx >= hpb->lu_pinned_start && rgn_idx <= hpb->lu_pinned_end) return true; return false; } static void ufshpb_kick_map_work(struct ufshpb_lu *hpb) { bool ret = false; unsigned long flags; if (ufshpb_get_state(hpb) != HPB_PRESENT) return; spin_lock_irqsave(&hpb->rsp_list_lock, flags); if (!list_empty(&hpb->lh_inact_rgn) || !list_empty(&hpb->lh_act_srgn)) ret = true; spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); if (ret) queue_work(ufshpb_wq, &hpb->map_work); } static bool ufshpb_is_hpb_rsp_valid(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, struct utp_hpb_rsp *rsp_field) { /* Check HPB_UPDATE_ALERT */ if (!(lrbp->ucd_rsp_ptr->header.dword_2 & UPIU_HEADER_DWORD(0, 2, 0, 0))) return false; if (be16_to_cpu(rsp_field->sense_data_len) != DEV_SENSE_SEG_LEN || rsp_field->desc_type != DEV_DES_TYPE || rsp_field->additional_len != DEV_ADDITIONAL_LEN || rsp_field->active_rgn_cnt > MAX_ACTIVE_NUM || rsp_field->inactive_rgn_cnt > MAX_INACTIVE_NUM || rsp_field->hpb_op == HPB_RSP_NONE || (rsp_field->hpb_op == HPB_RSP_REQ_REGION_UPDATE && !rsp_field->active_rgn_cnt && !rsp_field->inactive_rgn_cnt)) return false; if (!ufshpb_is_general_lun(rsp_field->lun)) { dev_warn(hba->dev, "ufshpb: lun(%d) not supported\n", lrbp->lun); return false; } return true; } static void ufshpb_iterate_rgn(struct ufshpb_lu *hpb, int rgn_idx, int srgn_idx, int srgn_offset, int cnt, bool set_dirty) { struct ufshpb_region *rgn; struct ufshpb_subregion *srgn, *prev_srgn = NULL; int set_bit_len; int bitmap_len; unsigned long flags; next_srgn: rgn = hpb->rgn_tbl + rgn_idx; srgn = rgn->srgn_tbl + srgn_idx; if (likely(!srgn->is_last)) bitmap_len = hpb->entries_per_srgn; else bitmap_len = hpb->last_srgn_entries; if ((srgn_offset + cnt) > bitmap_len) set_bit_len = bitmap_len - srgn_offset; else set_bit_len = cnt; spin_lock_irqsave(&hpb->rgn_state_lock, flags); if (rgn->rgn_state != HPB_RGN_INACTIVE) { if (set_dirty) { if (srgn->srgn_state == HPB_SRGN_VALID) bitmap_set(srgn->mctx->ppn_dirty, srgn_offset, set_bit_len); } else if (hpb->is_hcm) { /* rewind the read timer for lru regions */ rgn->read_timeout = ktime_add_ms(ktime_get(), rgn->hpb->params.read_timeout_ms); rgn->read_timeout_expiries = rgn->hpb->params.read_timeout_expiries; } } spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); if (hpb->is_hcm && prev_srgn != srgn) { bool activate = false; spin_lock(&rgn->rgn_lock); if (set_dirty) { rgn->reads -= srgn->reads; srgn->reads = 0; set_bit(RGN_FLAG_DIRTY, &rgn->rgn_flags); } else { srgn->reads++; rgn->reads++; if (srgn->reads == hpb->params.activation_thld) activate = true; } spin_unlock(&rgn->rgn_lock); if (activate || test_and_clear_bit(RGN_FLAG_UPDATE, &rgn->rgn_flags)) { spin_lock_irqsave(&hpb->rsp_list_lock, flags); ufshpb_update_active_info(hpb, rgn_idx, srgn_idx); spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, "activate region %d-%d\n", rgn_idx, srgn_idx); } prev_srgn = srgn; } srgn_offset = 0; if (++srgn_idx == hpb->srgns_per_rgn) { srgn_idx = 0; rgn_idx++; } cnt -= set_bit_len; if (cnt > 0) goto next_srgn; } static bool ufshpb_test_ppn_dirty(struct ufshpb_lu *hpb, int rgn_idx, int srgn_idx, int srgn_offset, int cnt) { struct ufshpb_region *rgn; struct ufshpb_subregion *srgn; int bitmap_len; int bit_len; next_srgn: rgn = hpb->rgn_tbl + rgn_idx; srgn = rgn->srgn_tbl + srgn_idx; if (likely(!srgn->is_last)) bitmap_len = hpb->entries_per_srgn; else bitmap_len = hpb->last_srgn_entries; if (!ufshpb_is_valid_srgn(rgn, srgn)) return true; /* * If the region state is active, mctx must be allocated. * In this case, check whether the region is evicted or * mctx allocation fail. */ if (unlikely(!srgn->mctx)) { dev_err(&hpb->sdev_ufs_lu->sdev_dev, "no mctx in region %d subregion %d.\n", srgn->rgn_idx, srgn->srgn_idx); return true; } if ((srgn_offset + cnt) > bitmap_len) bit_len = bitmap_len - srgn_offset; else bit_len = cnt; if (find_next_bit(srgn->mctx->ppn_dirty, bit_len + srgn_offset, srgn_offset) < bit_len + srgn_offset) return true; srgn_offset = 0; if (++srgn_idx == hpb->srgns_per_rgn) { srgn_idx = 0; rgn_idx++; } cnt -= bit_len; if (cnt > 0) goto next_srgn; return false; } static inline bool is_rgn_dirty(struct ufshpb_region *rgn) { return test_bit(RGN_FLAG_DIRTY, &rgn->rgn_flags); } static int ufshpb_fill_ppn_from_page(struct ufshpb_lu *hpb, struct ufshpb_map_ctx *mctx, int pos, int len, __be64 *ppn_buf) { struct page *page; int index, offset; int copied; index = pos / (PAGE_SIZE / HPB_ENTRY_SIZE); offset = pos % (PAGE_SIZE / HPB_ENTRY_SIZE); if ((offset + len) <= (PAGE_SIZE / HPB_ENTRY_SIZE)) copied = len; else copied = (PAGE_SIZE / HPB_ENTRY_SIZE) - offset; page = mctx->m_page[index]; if (unlikely(!page)) { dev_err(&hpb->sdev_ufs_lu->sdev_dev, "error. cannot find page in mctx\n"); return -ENOMEM; } memcpy(ppn_buf, page_address(page) + (offset * HPB_ENTRY_SIZE), copied * HPB_ENTRY_SIZE); return copied; } static void ufshpb_get_pos_from_lpn(struct ufshpb_lu *hpb, unsigned long lpn, int *rgn_idx, int *srgn_idx, int *offset) { int rgn_offset; *rgn_idx = lpn >> hpb->entries_per_rgn_shift; rgn_offset = lpn & hpb->entries_per_rgn_mask; *srgn_idx = rgn_offset >> hpb->entries_per_srgn_shift; *offset = rgn_offset & hpb->entries_per_srgn_mask; } static void ufshpb_set_hpb_read_to_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, __be64 ppn, u8 transfer_len) { unsigned char *cdb = lrbp->cmd->cmnd; __be64 ppn_tmp = ppn; cdb[0] = UFSHPB_READ; if (hba->dev_quirks & UFS_DEVICE_QUIRK_SWAP_L2P_ENTRY_FOR_HPB_READ) ppn_tmp = swab64(ppn); /* ppn value is stored as big-endian in the host memory */ memcpy(&cdb[6], &ppn_tmp, sizeof(__be64)); cdb[14] = transfer_len; cdb[15] = 0; lrbp->cmd->cmd_len = UFS_CDB_SIZE; } /* * This function will set up HPB read command using host-side L2P map data. */ int ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { struct ufshpb_lu *hpb; struct ufshpb_region *rgn; struct ufshpb_subregion *srgn; struct scsi_cmnd *cmd = lrbp->cmd; u32 lpn; __be64 ppn; unsigned long flags; int transfer_len, rgn_idx, srgn_idx, srgn_offset; int err = 0; hpb = ufshpb_get_hpb_data(cmd->device); if (!hpb) return -ENODEV; if (ufshpb_get_state(hpb) == HPB_INIT) return -ENODEV; if (ufshpb_get_state(hpb) != HPB_PRESENT) { dev_notice(&hpb->sdev_ufs_lu->sdev_dev, "%s: ufshpb state is not PRESENT", __func__); return -ENODEV; } if (blk_rq_is_passthrough(scsi_cmd_to_rq(cmd)) || (!ufshpb_is_write_or_discard(cmd) && !ufshpb_is_read_cmd(cmd))) return 0; transfer_len = sectors_to_logical(cmd->device, blk_rq_sectors(scsi_cmd_to_rq(cmd))); if (unlikely(!transfer_len)) return 0; lpn = sectors_to_logical(cmd->device, blk_rq_pos(scsi_cmd_to_rq(cmd))); ufshpb_get_pos_from_lpn(hpb, lpn, &rgn_idx, &srgn_idx, &srgn_offset); rgn = hpb->rgn_tbl + rgn_idx; srgn = rgn->srgn_tbl + srgn_idx; /* If command type is WRITE or DISCARD, set bitmap as drity */ if (ufshpb_is_write_or_discard(cmd)) { ufshpb_iterate_rgn(hpb, rgn_idx, srgn_idx, srgn_offset, transfer_len, true); return 0; } if (!ufshpb_is_supported_chunk(hpb, transfer_len)) return 0; WARN_ON_ONCE(transfer_len > HPB_MULTI_CHUNK_HIGH); if (hpb->is_hcm) { /* * in host control mode, reads are the main source for * activation trials. */ ufshpb_iterate_rgn(hpb, rgn_idx, srgn_idx, srgn_offset, transfer_len, false); /* keep those counters normalized */ if (rgn->reads > hpb->entries_per_srgn) schedule_work(&hpb->ufshpb_normalization_work); } spin_lock_irqsave(&hpb->rgn_state_lock, flags); if (ufshpb_test_ppn_dirty(hpb, rgn_idx, srgn_idx, srgn_offset, transfer_len)) { hpb->stats.miss_cnt++; spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); return 0; } err = ufshpb_fill_ppn_from_page(hpb, srgn->mctx, srgn_offset, 1, &ppn); spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); if (unlikely(err < 0)) { /* * In this case, the region state is active, * but the ppn table is not allocated. * Make sure that ppn table must be allocated on * active state. */ dev_err(hba->dev, "get ppn failed. err %d\n", err); return err; } ufshpb_set_hpb_read_to_upiu(hba, lrbp, ppn, transfer_len); hpb->stats.hit_cnt++; return 0; } static struct ufshpb_req *ufshpb_get_req(struct ufshpb_lu *hpb, int rgn_idx, enum req_opf dir, bool atomic) { struct ufshpb_req *rq; struct request *req; int retries = HPB_MAP_REQ_RETRIES; rq = kmem_cache_alloc(hpb->map_req_cache, GFP_KERNEL); if (!rq) return NULL; retry: req = blk_get_request(hpb->sdev_ufs_lu->request_queue, dir, BLK_MQ_REQ_NOWAIT); if (!atomic && (PTR_ERR(req) == -EWOULDBLOCK) && (--retries > 0)) { usleep_range(3000, 3100); goto retry; } if (IS_ERR(req)) goto free_rq; rq->hpb = hpb; rq->req = req; rq->rb.rgn_idx = rgn_idx; return rq; free_rq: kmem_cache_free(hpb->map_req_cache, rq); return NULL; } static void ufshpb_put_req(struct ufshpb_lu *hpb, struct ufshpb_req *rq) { blk_put_request(rq->req); kmem_cache_free(hpb->map_req_cache, rq); } static struct ufshpb_req *ufshpb_get_map_req(struct ufshpb_lu *hpb, struct ufshpb_subregion *srgn) { struct ufshpb_req *map_req; struct bio *bio; unsigned long flags; if (hpb->is_hcm && hpb->num_inflight_map_req >= hpb->params.inflight_map_req) { dev_info(&hpb->sdev_ufs_lu->sdev_dev, "map_req throttle. inflight %d throttle %d", hpb->num_inflight_map_req, hpb->params.inflight_map_req); return NULL; } map_req = ufshpb_get_req(hpb, srgn->rgn_idx, REQ_OP_DRV_IN, false); if (!map_req) return NULL; bio = bio_alloc(GFP_KERNEL, hpb->pages_per_srgn); if (!bio) { ufshpb_put_req(hpb, map_req); return NULL; } map_req->bio = bio; map_req->rb.srgn_idx = srgn->srgn_idx; map_req->rb.mctx = srgn->mctx; spin_lock_irqsave(&hpb->param_lock, flags); hpb->num_inflight_map_req++; spin_unlock_irqrestore(&hpb->param_lock, flags); return map_req; } static void ufshpb_put_map_req(struct ufshpb_lu *hpb, struct ufshpb_req *map_req) { unsigned long flags; bio_put(map_req->bio); ufshpb_put_req(hpb, map_req); spin_lock_irqsave(&hpb->param_lock, flags); hpb->num_inflight_map_req--; spin_unlock_irqrestore(&hpb->param_lock, flags); } static int ufshpb_clear_dirty_bitmap(struct ufshpb_lu *hpb, struct ufshpb_subregion *srgn) { struct ufshpb_region *rgn; u32 num_entries = hpb->entries_per_srgn; if (!srgn->mctx) { dev_err(&hpb->sdev_ufs_lu->sdev_dev, "no mctx in region %d subregion %d.\n", srgn->rgn_idx, srgn->srgn_idx); return -1; } if (unlikely(srgn->is_last)) num_entries = hpb->last_srgn_entries; bitmap_zero(srgn->mctx->ppn_dirty, num_entries); rgn = hpb->rgn_tbl + srgn->rgn_idx; clear_bit(RGN_FLAG_DIRTY, &rgn->rgn_flags); return 0; } static void ufshpb_update_active_info(struct ufshpb_lu *hpb, int rgn_idx, int srgn_idx) { struct ufshpb_region *rgn; struct ufshpb_subregion *srgn; rgn = hpb->rgn_tbl + rgn_idx; srgn = rgn->srgn_tbl + srgn_idx; list_del_init(&rgn->list_inact_rgn); if (list_empty(&srgn->list_act_srgn)) list_add_tail(&srgn->list_act_srgn, &hpb->lh_act_srgn); hpb->stats.rb_active_cnt++; } static void ufshpb_update_inactive_info(struct ufshpb_lu *hpb, int rgn_idx) { struct ufshpb_region *rgn; struct ufshpb_subregion *srgn; int srgn_idx; rgn = hpb->rgn_tbl + rgn_idx; for_each_sub_region(rgn, srgn_idx, srgn) list_del_init(&srgn->list_act_srgn); if (list_empty(&rgn->list_inact_rgn)) list_add_tail(&rgn->list_inact_rgn, &hpb->lh_inact_rgn); hpb->stats.rb_inactive_cnt++; } static void ufshpb_activate_subregion(struct ufshpb_lu *hpb, struct ufshpb_subregion *srgn) { struct ufshpb_region *rgn; /* * If there is no mctx in subregion * after I/O progress for HPB_READ_BUFFER, the region to which the * subregion belongs was evicted. * Make sure the region must not evict in I/O progress */ if (!srgn->mctx) { dev_err(&hpb->sdev_ufs_lu->sdev_dev, "no mctx in region %d subregion %d.\n", srgn->rgn_idx, srgn->srgn_idx); srgn->srgn_state = HPB_SRGN_INVALID; return; } rgn = hpb->rgn_tbl + srgn->rgn_idx; if (unlikely(rgn->rgn_state == HPB_RGN_INACTIVE)) { dev_err(&hpb->sdev_ufs_lu->sdev_dev, "region %d subregion %d evicted\n", srgn->rgn_idx, srgn->srgn_idx); srgn->srgn_state = HPB_SRGN_INVALID; return; } srgn->srgn_state = HPB_SRGN_VALID; } static void ufshpb_umap_req_compl_fn(struct request *req, blk_status_t error) { struct ufshpb_req *umap_req = (struct ufshpb_req *)req->end_io_data; ufshpb_put_req(umap_req->hpb, umap_req); } static void ufshpb_map_req_compl_fn(struct request *req, blk_status_t error) { struct ufshpb_req *map_req = (struct ufshpb_req *) req->end_io_data; struct ufshpb_lu *hpb = map_req->hpb; struct ufshpb_subregion *srgn; unsigned long flags; srgn = hpb->rgn_tbl[map_req->rb.rgn_idx].srgn_tbl + map_req->rb.srgn_idx; ufshpb_clear_dirty_bitmap(hpb, srgn); spin_lock_irqsave(&hpb->rgn_state_lock, flags); ufshpb_activate_subregion(hpb, srgn); spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); ufshpb_put_map_req(map_req->hpb, map_req); } static void ufshpb_set_unmap_cmd(unsigned char *cdb, struct ufshpb_region *rgn) { cdb[0] = UFSHPB_WRITE_BUFFER; cdb[1] = rgn ? UFSHPB_WRITE_BUFFER_INACT_SINGLE_ID : UFSHPB_WRITE_BUFFER_INACT_ALL_ID; if (rgn) put_unaligned_be16(rgn->rgn_idx, &cdb[2]); cdb[9] = 0x00; } static void ufshpb_set_read_buf_cmd(unsigned char *cdb, int rgn_idx, int srgn_idx, int srgn_mem_size) { cdb[0] = UFSHPB_READ_BUFFER; cdb[1] = UFSHPB_READ_BUFFER_ID; put_unaligned_be16(rgn_idx, &cdb[2]); put_unaligned_be16(srgn_idx, &cdb[4]); put_unaligned_be24(srgn_mem_size, &cdb[6]); cdb[9] = 0x00; } static void ufshpb_execute_umap_req(struct ufshpb_lu *hpb, struct ufshpb_req *umap_req, struct ufshpb_region *rgn) { struct request *req; struct scsi_request *rq; req = umap_req->req; req->timeout = 0; req->end_io_data = (void *)umap_req; rq = scsi_req(req); ufshpb_set_unmap_cmd(rq->cmd, rgn); rq->cmd_len = HPB_WRITE_BUFFER_CMD_LENGTH; blk_execute_rq_nowait(NULL, req, 1, ufshpb_umap_req_compl_fn); hpb->stats.umap_req_cnt++; } static int ufshpb_execute_map_req(struct ufshpb_lu *hpb, struct ufshpb_req *map_req, bool last) { struct request_queue *q; struct request *req; struct scsi_request *rq; int mem_size = hpb->srgn_mem_size; int ret = 0; int i; q = hpb->sdev_ufs_lu->request_queue; for (i = 0; i < hpb->pages_per_srgn; i++) { ret = bio_add_pc_page(q, map_req->bio, map_req->rb.mctx->m_page[i], PAGE_SIZE, 0); if (ret != PAGE_SIZE) { dev_err(&hpb->sdev_ufs_lu->sdev_dev, "bio_add_pc_page fail %d - %d\n", map_req->rb.rgn_idx, map_req->rb.srgn_idx); return ret; } } req = map_req->req; blk_rq_append_bio(req, map_req->bio); req->end_io_data = map_req; rq = scsi_req(req); if (unlikely(last)) mem_size = hpb->last_srgn_entries * HPB_ENTRY_SIZE; ufshpb_set_read_buf_cmd(rq->cmd, map_req->rb.rgn_idx, map_req->rb.srgn_idx, mem_size); rq->cmd_len = HPB_READ_BUFFER_CMD_LENGTH; blk_execute_rq_nowait(NULL, req, 1, ufshpb_map_req_compl_fn); hpb->stats.map_req_cnt++; return 0; } static struct ufshpb_map_ctx *ufshpb_get_map_ctx(struct ufshpb_lu *hpb, bool last) { struct ufshpb_map_ctx *mctx; u32 num_entries = hpb->entries_per_srgn; int i, j; mctx = mempool_alloc(ufshpb_mctx_pool, GFP_KERNEL); if (!mctx) return NULL; mctx->m_page = kmem_cache_alloc(hpb->m_page_cache, GFP_KERNEL); if (!mctx->m_page) goto release_mctx; if (unlikely(last)) num_entries = hpb->last_srgn_entries; mctx->ppn_dirty = bitmap_zalloc(num_entries, GFP_KERNEL); if (!mctx->ppn_dirty) goto release_m_page; for (i = 0; i < hpb->pages_per_srgn; i++) { mctx->m_page[i] = mempool_alloc(ufshpb_page_pool, GFP_KERNEL); if (!mctx->m_page[i]) { for (j = 0; j < i; j++) mempool_free(mctx->m_page[j], ufshpb_page_pool); goto release_ppn_dirty; } clear_page(page_address(mctx->m_page[i])); } return mctx; release_ppn_dirty: bitmap_free(mctx->ppn_dirty); release_m_page: kmem_cache_free(hpb->m_page_cache, mctx->m_page); release_mctx: mempool_free(mctx, ufshpb_mctx_pool); return NULL; } static void ufshpb_put_map_ctx(struct ufshpb_lu *hpb, struct ufshpb_map_ctx *mctx) { int i; for (i = 0; i < hpb->pages_per_srgn; i++) mempool_free(mctx->m_page[i], ufshpb_page_pool); bitmap_free(mctx->ppn_dirty); kmem_cache_free(hpb->m_page_cache, mctx->m_page); mempool_free(mctx, ufshpb_mctx_pool); } static int ufshpb_check_srgns_issue_state(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) { struct ufshpb_subregion *srgn; int srgn_idx; for_each_sub_region(rgn, srgn_idx, srgn) if (srgn->srgn_state == HPB_SRGN_ISSUED) return -EPERM; return 0; } static void ufshpb_read_to_handler(struct work_struct *work) { struct ufshpb_lu *hpb = container_of(work, struct ufshpb_lu, ufshpb_read_to_work.work); struct victim_select_info *lru_info = &hpb->lru_info; struct ufshpb_region *rgn, *next_rgn; unsigned long flags; unsigned int poll; LIST_HEAD(expired_list); if (test_and_set_bit(TIMEOUT_WORK_RUNNING, &hpb->work_data_bits)) return; spin_lock_irqsave(&hpb->rgn_state_lock, flags); list_for_each_entry_safe(rgn, next_rgn, &lru_info->lh_lru_rgn, list_lru_rgn) { bool timedout = ktime_after(ktime_get(), rgn->read_timeout); if (timedout) { rgn->read_timeout_expiries--; if (is_rgn_dirty(rgn) || rgn->read_timeout_expiries == 0) list_add(&rgn->list_expired_rgn, &expired_list); else rgn->read_timeout = ktime_add_ms(ktime_get(), hpb->params.read_timeout_ms); } } spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); list_for_each_entry_safe(rgn, next_rgn, &expired_list, list_expired_rgn) { list_del_init(&rgn->list_expired_rgn); spin_lock_irqsave(&hpb->rsp_list_lock, flags); ufshpb_update_inactive_info(hpb, rgn->rgn_idx); spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); } ufshpb_kick_map_work(hpb); clear_bit(TIMEOUT_WORK_RUNNING, &hpb->work_data_bits); poll = hpb->params.timeout_polling_interval_ms; schedule_delayed_work(&hpb->ufshpb_read_to_work, msecs_to_jiffies(poll)); } static void ufshpb_add_lru_info(struct victim_select_info *lru_info, struct ufshpb_region *rgn) { rgn->rgn_state = HPB_RGN_ACTIVE; list_add_tail(&rgn->list_lru_rgn, &lru_info->lh_lru_rgn); atomic_inc(&lru_info->active_cnt); if (rgn->hpb->is_hcm) { rgn->read_timeout = ktime_add_ms(ktime_get(), rgn->hpb->params.read_timeout_ms); rgn->read_timeout_expiries = rgn->hpb->params.read_timeout_expiries; } } static void ufshpb_hit_lru_info(struct victim_select_info *lru_info, struct ufshpb_region *rgn) { list_move_tail(&rgn->list_lru_rgn, &lru_info->lh_lru_rgn); } static struct ufshpb_region *ufshpb_victim_lru_info(struct ufshpb_lu *hpb) { struct victim_select_info *lru_info = &hpb->lru_info; struct ufshpb_region *rgn, *victim_rgn = NULL; list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) { if (!rgn) { dev_err(&hpb->sdev_ufs_lu->sdev_dev, "%s: no region allocated\n", __func__); return NULL; } if (ufshpb_check_srgns_issue_state(hpb, rgn)) continue; /* * in host control mode, verify that the exiting region * has fewer reads */ if (hpb->is_hcm && rgn->reads > hpb->params.eviction_thld_exit) continue; victim_rgn = rgn; break; } return victim_rgn; } static void ufshpb_cleanup_lru_info(struct victim_select_info *lru_info, struct ufshpb_region *rgn) { list_del_init(&rgn->list_lru_rgn); rgn->rgn_state = HPB_RGN_INACTIVE; atomic_dec(&lru_info->active_cnt); } static void ufshpb_purge_active_subregion(struct ufshpb_lu *hpb, struct ufshpb_subregion *srgn) { if (srgn->srgn_state != HPB_SRGN_UNUSED) { ufshpb_put_map_ctx(hpb, srgn->mctx); srgn->srgn_state = HPB_SRGN_UNUSED; srgn->mctx = NULL; } } static int ufshpb_issue_umap_req(struct ufshpb_lu *hpb, struct ufshpb_region *rgn, bool atomic) { struct ufshpb_req *umap_req; int rgn_idx = rgn ? rgn->rgn_idx : 0; umap_req = ufshpb_get_req(hpb, rgn_idx, REQ_OP_DRV_OUT, atomic); if (!umap_req) return -ENOMEM; ufshpb_execute_umap_req(hpb, umap_req, rgn); return 0; } static int ufshpb_issue_umap_single_req(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) { return ufshpb_issue_umap_req(hpb, rgn, true); } static int ufshpb_issue_umap_all_req(struct ufshpb_lu *hpb) { return ufshpb_issue_umap_req(hpb, NULL, false); } static void __ufshpb_evict_region(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) { struct victim_select_info *lru_info; struct ufshpb_subregion *srgn; int srgn_idx; lru_info = &hpb->lru_info; dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, "evict region %d\n", rgn->rgn_idx); ufshpb_cleanup_lru_info(lru_info, rgn); for_each_sub_region(rgn, srgn_idx, srgn) ufshpb_purge_active_subregion(hpb, srgn); } static int ufshpb_evict_region(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) { unsigned long flags; int ret = 0; spin_lock_irqsave(&hpb->rgn_state_lock, flags); if (rgn->rgn_state == HPB_RGN_PINNED) { dev_warn(&hpb->sdev_ufs_lu->sdev_dev, "pinned region cannot drop-out. region %d\n", rgn->rgn_idx); goto out; } if (!list_empty(&rgn->list_lru_rgn)) { if (ufshpb_check_srgns_issue_state(hpb, rgn)) { ret = -EBUSY; goto out; } if (hpb->is_hcm) { spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); ret = ufshpb_issue_umap_single_req(hpb, rgn); spin_lock_irqsave(&hpb->rgn_state_lock, flags); if (ret) goto out; } __ufshpb_evict_region(hpb, rgn); } out: spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); return ret; } static int ufshpb_issue_map_req(struct ufshpb_lu *hpb, struct ufshpb_region *rgn, struct ufshpb_subregion *srgn) { struct ufshpb_req *map_req; unsigned long flags; int ret; int err = -EAGAIN; bool alloc_required = false; enum HPB_SRGN_STATE state = HPB_SRGN_INVALID; spin_lock_irqsave(&hpb->rgn_state_lock, flags); if (ufshpb_get_state(hpb) != HPB_PRESENT) { dev_notice(&hpb->sdev_ufs_lu->sdev_dev, "%s: ufshpb state is not PRESENT\n", __func__); goto unlock_out; } if ((rgn->rgn_state == HPB_RGN_INACTIVE) && (srgn->srgn_state == HPB_SRGN_INVALID)) { err = 0; goto unlock_out; } if (srgn->srgn_state == HPB_SRGN_UNUSED) alloc_required = true; /* * If the subregion is already ISSUED state, * a specific event (e.g., GC or wear-leveling, etc.) occurs in * the device and HPB response for map loading is received. * In this case, after finishing the HPB_READ_BUFFER, * the next HPB_READ_BUFFER is performed again to obtain the latest * map data. */ if (srgn->srgn_state == HPB_SRGN_ISSUED) goto unlock_out; srgn->srgn_state = HPB_SRGN_ISSUED; spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); if (alloc_required) { srgn->mctx = ufshpb_get_map_ctx(hpb, srgn->is_last); if (!srgn->mctx) { dev_err(&hpb->sdev_ufs_lu->sdev_dev, "get map_ctx failed. region %d - %d\n", rgn->rgn_idx, srgn->srgn_idx); state = HPB_SRGN_UNUSED; goto change_srgn_state; } } map_req = ufshpb_get_map_req(hpb, srgn); if (!map_req) goto change_srgn_state; ret = ufshpb_execute_map_req(hpb, map_req, srgn->is_last); if (ret) { dev_err(&hpb->sdev_ufs_lu->sdev_dev, "%s: issue map_req failed: %d, region %d - %d\n", __func__, ret, srgn->rgn_idx, srgn->srgn_idx); goto free_map_req; } return 0; free_map_req: ufshpb_put_map_req(hpb, map_req); change_srgn_state: spin_lock_irqsave(&hpb->rgn_state_lock, flags); srgn->srgn_state = state; unlock_out: spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); return err; } static int ufshpb_add_region(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) { struct ufshpb_region *victim_rgn = NULL; struct victim_select_info *lru_info = &hpb->lru_info; unsigned long flags; int ret = 0; spin_lock_irqsave(&hpb->rgn_state_lock, flags); /* * If region belongs to lru_list, just move the region * to the front of lru list because the state of the region * is already active-state. */ if (!list_empty(&rgn->list_lru_rgn)) { ufshpb_hit_lru_info(lru_info, rgn); goto out; } if (rgn->rgn_state == HPB_RGN_INACTIVE) { if (atomic_read(&lru_info->active_cnt) == lru_info->max_lru_active_cnt) { /* * If the maximum number of active regions * is exceeded, evict the least recently used region. * This case may occur when the device responds * to the eviction information late. * It is okay to evict the least recently used region, * because the device could detect this region * by not issuing HPB_READ * * in host control mode, verify that the entering * region has enough reads */ if (hpb->is_hcm && rgn->reads < hpb->params.eviction_thld_enter) { ret = -EACCES; goto out; } victim_rgn = ufshpb_victim_lru_info(hpb); if (!victim_rgn) { dev_warn(&hpb->sdev_ufs_lu->sdev_dev, "cannot get victim region %s\n", hpb->is_hcm ? "" : "error"); ret = -ENOMEM; goto out; } dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, "LRU full (%d), choose victim %d\n", atomic_read(&lru_info->active_cnt), victim_rgn->rgn_idx); if (hpb->is_hcm) { spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); ret = ufshpb_issue_umap_single_req(hpb, victim_rgn); spin_lock_irqsave(&hpb->rgn_state_lock, flags); if (ret) goto out; } __ufshpb_evict_region(hpb, victim_rgn); } /* * When a region is added to lru_info list_head, * it is guaranteed that the subregion has been * assigned all mctx. If failed, try to receive mctx again * without being added to lru_info list_head */ ufshpb_add_lru_info(lru_info, rgn); } out: spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); return ret; } static void ufshpb_rsp_req_region_update(struct ufshpb_lu *hpb, struct utp_hpb_rsp *rsp_field) { struct ufshpb_region *rgn; struct ufshpb_subregion *srgn; int i, rgn_i, srgn_i; BUILD_BUG_ON(sizeof(struct ufshpb_active_field) != HPB_ACT_FIELD_SIZE); /* * If the active region and the inactive region are the same, * we will inactivate this region. * The device could check this (region inactivated) and * will response the proper active region information */ for (i = 0; i < rsp_field->active_rgn_cnt; i++) { rgn_i = be16_to_cpu(rsp_field->hpb_active_field[i].active_rgn); srgn_i = be16_to_cpu(rsp_field->hpb_active_field[i].active_srgn); rgn = hpb->rgn_tbl + rgn_i; if (hpb->is_hcm && (rgn->rgn_state != HPB_RGN_ACTIVE || is_rgn_dirty(rgn))) { /* * in host control mode, subregion activation * recommendations are only allowed to active regions. * Also, ignore recommendations for dirty regions - the * host will make decisions concerning those by himself */ continue; } dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, "activate(%d) region %d - %d\n", i, rgn_i, srgn_i); spin_lock(&hpb->rsp_list_lock); ufshpb_update_active_info(hpb, rgn_i, srgn_i); spin_unlock(&hpb->rsp_list_lock); srgn = rgn->srgn_tbl + srgn_i; /* blocking HPB_READ */ spin_lock(&hpb->rgn_state_lock); if (srgn->srgn_state == HPB_SRGN_VALID) srgn->srgn_state = HPB_SRGN_INVALID; spin_unlock(&hpb->rgn_state_lock); } if (hpb->is_hcm) { /* * in host control mode the device is not allowed to inactivate * regions */ goto out; } for (i = 0; i < rsp_field->inactive_rgn_cnt; i++) { rgn_i = be16_to_cpu(rsp_field->hpb_inactive_field[i]); dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, "inactivate(%d) region %d\n", i, rgn_i); spin_lock(&hpb->rsp_list_lock); ufshpb_update_inactive_info(hpb, rgn_i); spin_unlock(&hpb->rsp_list_lock); rgn = hpb->rgn_tbl + rgn_i; spin_lock(&hpb->rgn_state_lock); if (rgn->rgn_state != HPB_RGN_INACTIVE) { for (srgn_i = 0; srgn_i < rgn->srgn_cnt; srgn_i++) { srgn = rgn->srgn_tbl + srgn_i; if (srgn->srgn_state == HPB_SRGN_VALID) srgn->srgn_state = HPB_SRGN_INVALID; } } spin_unlock(&hpb->rgn_state_lock); } out: dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, "Noti: #ACT %u #INACT %u\n", rsp_field->active_rgn_cnt, rsp_field->inactive_rgn_cnt); if (ufshpb_get_state(hpb) == HPB_PRESENT) queue_work(ufshpb_wq, &hpb->map_work); } static void ufshpb_dev_reset_handler(struct ufshpb_lu *hpb) { struct victim_select_info *lru_info = &hpb->lru_info; struct ufshpb_region *rgn; unsigned long flags; spin_lock_irqsave(&hpb->rgn_state_lock, flags); list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) set_bit(RGN_FLAG_UPDATE, &rgn->rgn_flags); spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); } /* * This function will parse recommended active subregion information in sense * data field of response UPIU with SAM_STAT_GOOD state. */ void ufshpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { struct ufshpb_lu *hpb = ufshpb_get_hpb_data(lrbp->cmd->device); struct utp_hpb_rsp *rsp_field = &lrbp->ucd_rsp_ptr->hr; int data_seg_len; if (unlikely(lrbp->lun != rsp_field->lun)) { struct scsi_device *sdev; bool found = false; __shost_for_each_device(sdev, hba->host) { hpb = ufshpb_get_hpb_data(sdev); if (!hpb) continue; if (rsp_field->lun == hpb->lun) { found = true; break; } } if (!found) return; } if (!hpb) return; if (ufshpb_get_state(hpb) == HPB_INIT) return; if ((ufshpb_get_state(hpb) != HPB_PRESENT) && (ufshpb_get_state(hpb) != HPB_SUSPEND)) { dev_notice(&hpb->sdev_ufs_lu->sdev_dev, "%s: ufshpb state is not PRESENT/SUSPEND\n", __func__); return; } data_seg_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & MASK_RSP_UPIU_DATA_SEG_LEN; /* To flush remained rsp_list, we queue the map_work task */ if (!data_seg_len) { if (!ufshpb_is_general_lun(hpb->lun)) return; ufshpb_kick_map_work(hpb); return; } BUILD_BUG_ON(sizeof(struct utp_hpb_rsp) != UTP_HPB_RSP_SIZE); if (!ufshpb_is_hpb_rsp_valid(hba, lrbp, rsp_field)) return; hpb->stats.rb_noti_cnt++; switch (rsp_field->hpb_op) { case HPB_RSP_REQ_REGION_UPDATE: if (data_seg_len != DEV_DATA_SEG_LEN) dev_warn(&hpb->sdev_ufs_lu->sdev_dev, "%s: data seg length is not same.\n", __func__); ufshpb_rsp_req_region_update(hpb, rsp_field); break; case HPB_RSP_DEV_RESET: dev_warn(&hpb->sdev_ufs_lu->sdev_dev, "UFS device lost HPB information during PM.\n"); if (hpb->is_hcm) { struct scsi_device *sdev; __shost_for_each_device(sdev, hba->host) { struct ufshpb_lu *h = sdev->hostdata; if (h) ufshpb_dev_reset_handler(h); } } break; default: dev_notice(&hpb->sdev_ufs_lu->sdev_dev, "hpb_op is not available: %d\n", rsp_field->hpb_op); break; } } static void ufshpb_add_active_list(struct ufshpb_lu *hpb, struct ufshpb_region *rgn, struct ufshpb_subregion *srgn) { if (!list_empty(&rgn->list_inact_rgn)) return; if (!list_empty(&srgn->list_act_srgn)) { list_move(&srgn->list_act_srgn, &hpb->lh_act_srgn); return; } list_add(&srgn->list_act_srgn, &hpb->lh_act_srgn); } static void ufshpb_add_pending_evict_list(struct ufshpb_lu *hpb, struct ufshpb_region *rgn, struct list_head *pending_list) { struct ufshpb_subregion *srgn; int srgn_idx; if (!list_empty(&rgn->list_inact_rgn)) return; for_each_sub_region(rgn, srgn_idx, srgn) if (!list_empty(&srgn->list_act_srgn)) return; list_add_tail(&rgn->list_inact_rgn, pending_list); } static void ufshpb_run_active_subregion_list(struct ufshpb_lu *hpb) { struct ufshpb_region *rgn; struct ufshpb_subregion *srgn; unsigned long flags; int ret = 0; spin_lock_irqsave(&hpb->rsp_list_lock, flags); while ((srgn = list_first_entry_or_null(&hpb->lh_act_srgn, struct ufshpb_subregion, list_act_srgn))) { if (ufshpb_get_state(hpb) == HPB_SUSPEND) break; list_del_init(&srgn->list_act_srgn); spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); rgn = hpb->rgn_tbl + srgn->rgn_idx; ret = ufshpb_add_region(hpb, rgn); if (ret) goto active_failed; ret = ufshpb_issue_map_req(hpb, rgn, srgn); if (ret) { dev_err(&hpb->sdev_ufs_lu->sdev_dev, "issue map_req failed. ret %d, region %d - %d\n", ret, rgn->rgn_idx, srgn->srgn_idx); goto active_failed; } spin_lock_irqsave(&hpb->rsp_list_lock, flags); } spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); return; active_failed: dev_err(&hpb->sdev_ufs_lu->sdev_dev, "failed to activate region %d - %d, will retry\n", rgn->rgn_idx, srgn->srgn_idx); spin_lock_irqsave(&hpb->rsp_list_lock, flags); ufshpb_add_active_list(hpb, rgn, srgn); spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); } static void ufshpb_run_inactive_region_list(struct ufshpb_lu *hpb) { struct ufshpb_region *rgn; unsigned long flags; int ret; LIST_HEAD(pending_list); spin_lock_irqsave(&hpb->rsp_list_lock, flags); while ((rgn = list_first_entry_or_null(&hpb->lh_inact_rgn, struct ufshpb_region, list_inact_rgn))) { if (ufshpb_get_state(hpb) == HPB_SUSPEND) break; list_del_init(&rgn->list_inact_rgn); spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); ret = ufshpb_evict_region(hpb, rgn); if (ret) { spin_lock_irqsave(&hpb->rsp_list_lock, flags); ufshpb_add_pending_evict_list(hpb, rgn, &pending_list); spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); } spin_lock_irqsave(&hpb->rsp_list_lock, flags); } list_splice(&pending_list, &hpb->lh_inact_rgn); spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); } static void ufshpb_normalization_work_handler(struct work_struct *work) { struct ufshpb_lu *hpb = container_of(work, struct ufshpb_lu, ufshpb_normalization_work); int rgn_idx; u8 factor = hpb->params.normalization_factor; for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) { struct ufshpb_region *rgn = hpb->rgn_tbl + rgn_idx; int srgn_idx; spin_lock(&rgn->rgn_lock); rgn->reads = 0; for (srgn_idx = 0; srgn_idx < hpb->srgns_per_rgn; srgn_idx++) { struct ufshpb_subregion *srgn = rgn->srgn_tbl + srgn_idx; srgn->reads >>= factor; rgn->reads += srgn->reads; } spin_unlock(&rgn->rgn_lock); if (rgn->rgn_state != HPB_RGN_ACTIVE || rgn->reads) continue; /* if region is active but has no reads - inactivate it */ spin_lock(&hpb->rsp_list_lock); ufshpb_update_inactive_info(hpb, rgn->rgn_idx); spin_unlock(&hpb->rsp_list_lock); } } static void ufshpb_map_work_handler(struct work_struct *work) { struct ufshpb_lu *hpb = container_of(work, struct ufshpb_lu, map_work); if (ufshpb_get_state(hpb) != HPB_PRESENT) { dev_notice(&hpb->sdev_ufs_lu->sdev_dev, "%s: ufshpb state is not PRESENT\n", __func__); return; } ufshpb_run_inactive_region_list(hpb); ufshpb_run_active_subregion_list(hpb); } /* * this function doesn't need to hold lock due to be called in init. * (rgn_state_lock, rsp_list_lock, etc..) */ static int ufshpb_init_pinned_active_region(struct ufs_hba *hba, struct ufshpb_lu *hpb, struct ufshpb_region *rgn) { struct ufshpb_subregion *srgn; int srgn_idx, i; int err = 0; for_each_sub_region(rgn, srgn_idx, srgn) { srgn->mctx = ufshpb_get_map_ctx(hpb, srgn->is_last); srgn->srgn_state = HPB_SRGN_INVALID; if (!srgn->mctx) { err = -ENOMEM; dev_err(hba->dev, "alloc mctx for pinned region failed\n"); goto release; } list_add_tail(&srgn->list_act_srgn, &hpb->lh_act_srgn); } rgn->rgn_state = HPB_RGN_PINNED; return 0; release: for (i = 0; i < srgn_idx; i++) { srgn = rgn->srgn_tbl + i; ufshpb_put_map_ctx(hpb, srgn->mctx); } return err; } static void ufshpb_init_subregion_tbl(struct ufshpb_lu *hpb, struct ufshpb_region *rgn, bool last) { int srgn_idx; struct ufshpb_subregion *srgn; for_each_sub_region(rgn, srgn_idx, srgn) { INIT_LIST_HEAD(&srgn->list_act_srgn); srgn->rgn_idx = rgn->rgn_idx; srgn->srgn_idx = srgn_idx; srgn->srgn_state = HPB_SRGN_UNUSED; } if (unlikely(last && hpb->last_srgn_entries)) srgn->is_last = true; } static int ufshpb_alloc_subregion_tbl(struct ufshpb_lu *hpb, struct ufshpb_region *rgn, int srgn_cnt) { rgn->srgn_tbl = kvcalloc(srgn_cnt, sizeof(struct ufshpb_subregion), GFP_KERNEL); if (!rgn->srgn_tbl) return -ENOMEM; rgn->srgn_cnt = srgn_cnt; return 0; } static void ufshpb_lu_parameter_init(struct ufs_hba *hba, struct ufshpb_lu *hpb, struct ufshpb_dev_info *hpb_dev_info, struct ufshpb_lu_info *hpb_lu_info) { u32 entries_per_rgn; u64 rgn_mem_size, tmp; if (ufshpb_is_legacy(hba)) hpb->pre_req_max_tr_len = HPB_LEGACY_CHUNK_HIGH; else hpb->pre_req_max_tr_len = HPB_MULTI_CHUNK_HIGH; hpb->lu_pinned_start = hpb_lu_info->pinned_start; hpb->lu_pinned_end = hpb_lu_info->num_pinned ? (hpb_lu_info->pinned_start + hpb_lu_info->num_pinned - 1) : PINNED_NOT_SET; hpb->lru_info.max_lru_active_cnt = hpb_lu_info->max_active_rgns - hpb_lu_info->num_pinned; rgn_mem_size = (1ULL << hpb_dev_info->rgn_size) * HPB_RGN_SIZE_UNIT * HPB_ENTRY_SIZE; do_div(rgn_mem_size, HPB_ENTRY_BLOCK_SIZE); hpb->srgn_mem_size = (1ULL << hpb_dev_info->srgn_size) * HPB_RGN_SIZE_UNIT / HPB_ENTRY_BLOCK_SIZE * HPB_ENTRY_SIZE; tmp = rgn_mem_size; do_div(tmp, HPB_ENTRY_SIZE); entries_per_rgn = (u32)tmp; hpb->entries_per_rgn_shift = ilog2(entries_per_rgn); hpb->entries_per_rgn_mask = entries_per_rgn - 1; hpb->entries_per_srgn = hpb->srgn_mem_size / HPB_ENTRY_SIZE; hpb->entries_per_srgn_shift = ilog2(hpb->entries_per_srgn); hpb->entries_per_srgn_mask = hpb->entries_per_srgn - 1; tmp = rgn_mem_size; do_div(tmp, hpb->srgn_mem_size); hpb->srgns_per_rgn = (int)tmp; hpb->rgns_per_lu = DIV_ROUND_UP(hpb_lu_info->num_blocks, entries_per_rgn); hpb->srgns_per_lu = DIV_ROUND_UP(hpb_lu_info->num_blocks, (hpb->srgn_mem_size / HPB_ENTRY_SIZE)); hpb->last_srgn_entries = hpb_lu_info->num_blocks % (hpb->srgn_mem_size / HPB_ENTRY_SIZE); hpb->pages_per_srgn = DIV_ROUND_UP(hpb->srgn_mem_size, PAGE_SIZE); if (hpb_dev_info->control_mode == HPB_HOST_CONTROL) hpb->is_hcm = true; } static int ufshpb_alloc_region_tbl(struct ufs_hba *hba, struct ufshpb_lu *hpb) { struct ufshpb_region *rgn_table, *rgn; int rgn_idx, i; int ret = 0; rgn_table = kvcalloc(hpb->rgns_per_lu, sizeof(struct ufshpb_region), GFP_KERNEL); if (!rgn_table) return -ENOMEM; for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) { int srgn_cnt = hpb->srgns_per_rgn; bool last_srgn = false; rgn = rgn_table + rgn_idx; rgn->rgn_idx = rgn_idx; spin_lock_init(&rgn->rgn_lock); INIT_LIST_HEAD(&rgn->list_inact_rgn); INIT_LIST_HEAD(&rgn->list_lru_rgn); INIT_LIST_HEAD(&rgn->list_expired_rgn); if (rgn_idx == hpb->rgns_per_lu - 1) { srgn_cnt = ((hpb->srgns_per_lu - 1) % hpb->srgns_per_rgn) + 1; last_srgn = true; } ret = ufshpb_alloc_subregion_tbl(hpb, rgn, srgn_cnt); if (ret) goto release_srgn_table; ufshpb_init_subregion_tbl(hpb, rgn, last_srgn); if (ufshpb_is_pinned_region(hpb, rgn_idx)) { ret = ufshpb_init_pinned_active_region(hba, hpb, rgn); if (ret) goto release_srgn_table; } else { rgn->rgn_state = HPB_RGN_INACTIVE; } rgn->rgn_flags = 0; rgn->hpb = hpb; } hpb->rgn_tbl = rgn_table; return 0; release_srgn_table: for (i = 0; i <= rgn_idx; i++) kvfree(rgn_table[i].srgn_tbl); kvfree(rgn_table); return ret; } static void ufshpb_destroy_subregion_tbl(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) { int srgn_idx; struct ufshpb_subregion *srgn; for_each_sub_region(rgn, srgn_idx, srgn) if (srgn->srgn_state != HPB_SRGN_UNUSED) { srgn->srgn_state = HPB_SRGN_UNUSED; ufshpb_put_map_ctx(hpb, srgn->mctx); } } static void ufshpb_destroy_region_tbl(struct ufshpb_lu *hpb) { int rgn_idx; for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) { struct ufshpb_region *rgn; rgn = hpb->rgn_tbl + rgn_idx; if (rgn->rgn_state != HPB_RGN_INACTIVE) { rgn->rgn_state = HPB_RGN_INACTIVE; ufshpb_destroy_subregion_tbl(hpb, rgn); } kvfree(rgn->srgn_tbl); } kvfree(hpb->rgn_tbl); } /* SYSFS functions */ #define ufshpb_sysfs_attr_show_func(__name) \ static ssize_t __name##_show(struct device *dev, \ struct device_attribute *attr, char *buf) \ { \ struct scsi_device *sdev = to_scsi_device(dev); \ struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); \ \ if (!hpb) \ return -ENODEV; \ \ return sysfs_emit(buf, "%llu\n", hpb->stats.__name); \ } \ \ static DEVICE_ATTR_RO(__name) ufshpb_sysfs_attr_show_func(hit_cnt); ufshpb_sysfs_attr_show_func(miss_cnt); ufshpb_sysfs_attr_show_func(rb_noti_cnt); ufshpb_sysfs_attr_show_func(rb_active_cnt); ufshpb_sysfs_attr_show_func(rb_inactive_cnt); ufshpb_sysfs_attr_show_func(map_req_cnt); ufshpb_sysfs_attr_show_func(umap_req_cnt); static struct attribute *hpb_dev_stat_attrs[] = { &dev_attr_hit_cnt.attr, &dev_attr_miss_cnt.attr, &dev_attr_rb_noti_cnt.attr, &dev_attr_rb_active_cnt.attr, &dev_attr_rb_inactive_cnt.attr, &dev_attr_map_req_cnt.attr, &dev_attr_umap_req_cnt.attr, NULL, }; struct attribute_group ufs_sysfs_hpb_stat_group = { .name = "hpb_stats", .attrs = hpb_dev_stat_attrs, }; /* SYSFS functions */ #define ufshpb_sysfs_param_show_func(__name) \ static ssize_t __name##_show(struct device *dev, \ struct device_attribute *attr, char *buf) \ { \ struct scsi_device *sdev = to_scsi_device(dev); \ struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); \ \ if (!hpb) \ return -ENODEV; \ \ return sysfs_emit(buf, "%d\n", hpb->params.__name); \ } ufshpb_sysfs_param_show_func(requeue_timeout_ms); static ssize_t requeue_timeout_ms_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct scsi_device *sdev = to_scsi_device(dev); struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); int val; if (!hpb) return -ENODEV; if (kstrtouint(buf, 0, &val)) return -EINVAL; if (val < 0) return -EINVAL; hpb->params.requeue_timeout_ms = val; return count; } static DEVICE_ATTR_RW(requeue_timeout_ms); ufshpb_sysfs_param_show_func(activation_thld); static ssize_t activation_thld_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct scsi_device *sdev = to_scsi_device(dev); struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); int val; if (!hpb) return -ENODEV; if (!hpb->is_hcm) return -EOPNOTSUPP; if (kstrtouint(buf, 0, &val)) return -EINVAL; if (val <= 0) return -EINVAL; hpb->params.activation_thld = val; return count; } static DEVICE_ATTR_RW(activation_thld); ufshpb_sysfs_param_show_func(normalization_factor); static ssize_t normalization_factor_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct scsi_device *sdev = to_scsi_device(dev); struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); int val; if (!hpb) return -ENODEV; if (!hpb->is_hcm) return -EOPNOTSUPP; if (kstrtouint(buf, 0, &val)) return -EINVAL; if (val <= 0 || val > ilog2(hpb->entries_per_srgn)) return -EINVAL; hpb->params.normalization_factor = val; return count; } static DEVICE_ATTR_RW(normalization_factor); ufshpb_sysfs_param_show_func(eviction_thld_enter); static ssize_t eviction_thld_enter_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct scsi_device *sdev = to_scsi_device(dev); struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); int val; if (!hpb) return -ENODEV; if (!hpb->is_hcm) return -EOPNOTSUPP; if (kstrtouint(buf, 0, &val)) return -EINVAL; if (val <= hpb->params.eviction_thld_exit) return -EINVAL; hpb->params.eviction_thld_enter = val; return count; } static DEVICE_ATTR_RW(eviction_thld_enter); ufshpb_sysfs_param_show_func(eviction_thld_exit); static ssize_t eviction_thld_exit_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct scsi_device *sdev = to_scsi_device(dev); struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); int val; if (!hpb) return -ENODEV; if (!hpb->is_hcm) return -EOPNOTSUPP; if (kstrtouint(buf, 0, &val)) return -EINVAL; if (val <= hpb->params.activation_thld) return -EINVAL; hpb->params.eviction_thld_exit = val; return count; } static DEVICE_ATTR_RW(eviction_thld_exit); ufshpb_sysfs_param_show_func(read_timeout_ms); static ssize_t read_timeout_ms_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct scsi_device *sdev = to_scsi_device(dev); struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); int val; if (!hpb) return -ENODEV; if (!hpb->is_hcm) return -EOPNOTSUPP; if (kstrtouint(buf, 0, &val)) return -EINVAL; /* read_timeout >> timeout_polling_interval */ if (val < hpb->params.timeout_polling_interval_ms * 2) return -EINVAL; hpb->params.read_timeout_ms = val; return count; } static DEVICE_ATTR_RW(read_timeout_ms); ufshpb_sysfs_param_show_func(read_timeout_expiries); static ssize_t read_timeout_expiries_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct scsi_device *sdev = to_scsi_device(dev); struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); int val; if (!hpb) return -ENODEV; if (!hpb->is_hcm) return -EOPNOTSUPP; if (kstrtouint(buf, 0, &val)) return -EINVAL; if (val <= 0) return -EINVAL; hpb->params.read_timeout_expiries = val; return count; } static DEVICE_ATTR_RW(read_timeout_expiries); ufshpb_sysfs_param_show_func(timeout_polling_interval_ms); static ssize_t timeout_polling_interval_ms_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct scsi_device *sdev = to_scsi_device(dev); struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); int val; if (!hpb) return -ENODEV; if (!hpb->is_hcm) return -EOPNOTSUPP; if (kstrtouint(buf, 0, &val)) return -EINVAL; /* timeout_polling_interval << read_timeout */ if (val <= 0 || val > hpb->params.read_timeout_ms / 2) return -EINVAL; hpb->params.timeout_polling_interval_ms = val; return count; } static DEVICE_ATTR_RW(timeout_polling_interval_ms); ufshpb_sysfs_param_show_func(inflight_map_req); static ssize_t inflight_map_req_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct scsi_device *sdev = to_scsi_device(dev); struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); int val; if (!hpb) return -ENODEV; if (!hpb->is_hcm) return -EOPNOTSUPP; if (kstrtouint(buf, 0, &val)) return -EINVAL; if (val <= 0 || val > hpb->sdev_ufs_lu->queue_depth - 1) return -EINVAL; hpb->params.inflight_map_req = val; return count; } static DEVICE_ATTR_RW(inflight_map_req); static void ufshpb_hcm_param_init(struct ufshpb_lu *hpb) { hpb->params.activation_thld = ACTIVATION_THRESHOLD; hpb->params.normalization_factor = 1; hpb->params.eviction_thld_enter = (ACTIVATION_THRESHOLD << 5); hpb->params.eviction_thld_exit = (ACTIVATION_THRESHOLD << 4); hpb->params.read_timeout_ms = READ_TO_MS; hpb->params.read_timeout_expiries = READ_TO_EXPIRIES; hpb->params.timeout_polling_interval_ms = POLLING_INTERVAL_MS; hpb->params.inflight_map_req = THROTTLE_MAP_REQ_DEFAULT; } static struct attribute *hpb_dev_param_attrs[] = { &dev_attr_requeue_timeout_ms.attr, &dev_attr_activation_thld.attr, &dev_attr_normalization_factor.attr, &dev_attr_eviction_thld_enter.attr, &dev_attr_eviction_thld_exit.attr, &dev_attr_read_timeout_ms.attr, &dev_attr_read_timeout_expiries.attr, &dev_attr_timeout_polling_interval_ms.attr, &dev_attr_inflight_map_req.attr, NULL, }; struct attribute_group ufs_sysfs_hpb_param_group = { .name = "hpb_params", .attrs = hpb_dev_param_attrs, }; static int ufshpb_pre_req_mempool_init(struct ufshpb_lu *hpb) { struct ufshpb_req *pre_req = NULL, *t; int qd = hpb->sdev_ufs_lu->queue_depth / 2; int i; INIT_LIST_HEAD(&hpb->lh_pre_req_free); hpb->pre_req = kcalloc(qd, sizeof(struct ufshpb_req), GFP_KERNEL); hpb->throttle_pre_req = qd; hpb->num_inflight_pre_req = 0; if (!hpb->pre_req) goto release_mem; for (i = 0; i < qd; i++) { pre_req = hpb->pre_req + i; INIT_LIST_HEAD(&pre_req->list_req); pre_req->req = NULL; pre_req->bio = bio_alloc(GFP_KERNEL, 1); if (!pre_req->bio) goto release_mem; pre_req->wb.m_page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!pre_req->wb.m_page) { bio_put(pre_req->bio); goto release_mem; } list_add_tail(&pre_req->list_req, &hpb->lh_pre_req_free); } return 0; release_mem: list_for_each_entry_safe(pre_req, t, &hpb->lh_pre_req_free, list_req) { list_del_init(&pre_req->list_req); bio_put(pre_req->bio); __free_page(pre_req->wb.m_page); } kfree(hpb->pre_req); return -ENOMEM; } static void ufshpb_pre_req_mempool_destroy(struct ufshpb_lu *hpb) { struct ufshpb_req *pre_req = NULL; int i; for (i = 0; i < hpb->throttle_pre_req; i++) { pre_req = hpb->pre_req + i; bio_put(hpb->pre_req[i].bio); if (!pre_req->wb.m_page) __free_page(hpb->pre_req[i].wb.m_page); list_del_init(&pre_req->list_req); } kfree(hpb->pre_req); } static void ufshpb_stat_init(struct ufshpb_lu *hpb) { hpb->stats.hit_cnt = 0; hpb->stats.miss_cnt = 0; hpb->stats.rb_noti_cnt = 0; hpb->stats.rb_active_cnt = 0; hpb->stats.rb_inactive_cnt = 0; hpb->stats.map_req_cnt = 0; hpb->stats.umap_req_cnt = 0; } static void ufshpb_param_init(struct ufshpb_lu *hpb) { hpb->params.requeue_timeout_ms = HPB_REQUEUE_TIME_MS; if (hpb->is_hcm) ufshpb_hcm_param_init(hpb); } static int ufshpb_lu_hpb_init(struct ufs_hba *hba, struct ufshpb_lu *hpb) { int ret; spin_lock_init(&hpb->rgn_state_lock); spin_lock_init(&hpb->rsp_list_lock); spin_lock_init(&hpb->param_lock); INIT_LIST_HEAD(&hpb->lru_info.lh_lru_rgn); INIT_LIST_HEAD(&hpb->lh_act_srgn); INIT_LIST_HEAD(&hpb->lh_inact_rgn); INIT_LIST_HEAD(&hpb->list_hpb_lu); INIT_WORK(&hpb->map_work, ufshpb_map_work_handler); if (hpb->is_hcm) { INIT_WORK(&hpb->ufshpb_normalization_work, ufshpb_normalization_work_handler); INIT_DELAYED_WORK(&hpb->ufshpb_read_to_work, ufshpb_read_to_handler); } hpb->map_req_cache = kmem_cache_create("ufshpb_req_cache", sizeof(struct ufshpb_req), 0, 0, NULL); if (!hpb->map_req_cache) { dev_err(hba->dev, "ufshpb(%d) ufshpb_req_cache create fail", hpb->lun); return -ENOMEM; } hpb->m_page_cache = kmem_cache_create("ufshpb_m_page_cache", sizeof(struct page *) * hpb->pages_per_srgn, 0, 0, NULL); if (!hpb->m_page_cache) { dev_err(hba->dev, "ufshpb(%d) ufshpb_m_page_cache create fail", hpb->lun); ret = -ENOMEM; goto release_req_cache; } ret = ufshpb_pre_req_mempool_init(hpb); if (ret) { dev_err(hba->dev, "ufshpb(%d) pre_req_mempool init fail", hpb->lun); goto release_m_page_cache; } ret = ufshpb_alloc_region_tbl(hba, hpb); if (ret) goto release_pre_req_mempool; ufshpb_stat_init(hpb); ufshpb_param_init(hpb); if (hpb->is_hcm) { unsigned int poll; poll = hpb->params.timeout_polling_interval_ms; schedule_delayed_work(&hpb->ufshpb_read_to_work, msecs_to_jiffies(poll)); } return 0; release_pre_req_mempool: ufshpb_pre_req_mempool_destroy(hpb); release_m_page_cache: kmem_cache_destroy(hpb->m_page_cache); release_req_cache: kmem_cache_destroy(hpb->map_req_cache); return ret; } static struct ufshpb_lu * ufshpb_alloc_hpb_lu(struct ufs_hba *hba, struct scsi_device *sdev, struct ufshpb_dev_info *hpb_dev_info, struct ufshpb_lu_info *hpb_lu_info) { struct ufshpb_lu *hpb; int ret; hpb = kzalloc(sizeof(struct ufshpb_lu), GFP_KERNEL); if (!hpb) return NULL; hpb->lun = sdev->lun; hpb->sdev_ufs_lu = sdev; ufshpb_lu_parameter_init(hba, hpb, hpb_dev_info, hpb_lu_info); ret = ufshpb_lu_hpb_init(hba, hpb); if (ret) { dev_err(hba->dev, "hpb lu init failed. ret %d", ret); goto release_hpb; } sdev->hostdata = hpb; return hpb; release_hpb: kfree(hpb); return NULL; } static void ufshpb_discard_rsp_lists(struct ufshpb_lu *hpb) { struct ufshpb_region *rgn, *next_rgn; struct ufshpb_subregion *srgn, *next_srgn; unsigned long flags; /* * If the device reset occurred, the remaining HPB region information * may be stale. Therefore, by discarding the lists of HPB response * that remained after reset, we prevent unnecessary work. */ spin_lock_irqsave(&hpb->rsp_list_lock, flags); list_for_each_entry_safe(rgn, next_rgn, &hpb->lh_inact_rgn, list_inact_rgn) list_del_init(&rgn->list_inact_rgn); list_for_each_entry_safe(srgn, next_srgn, &hpb->lh_act_srgn, list_act_srgn) list_del_init(&srgn->list_act_srgn); spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); } static void ufshpb_cancel_jobs(struct ufshpb_lu *hpb) { if (hpb->is_hcm) { cancel_delayed_work_sync(&hpb->ufshpb_read_to_work); cancel_work_sync(&hpb->ufshpb_normalization_work); } cancel_work_sync(&hpb->map_work); } static bool ufshpb_check_hpb_reset_query(struct ufs_hba *hba) { int err = 0; bool flag_res = true; int try; /* wait for the device to complete HPB reset query */ for (try = 0; try < HPB_RESET_REQ_RETRIES; try++) { dev_dbg(hba->dev, "%s start flag reset polling %d times\n", __func__, try); /* Poll fHpbReset flag to be cleared */ err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG, QUERY_FLAG_IDN_HPB_RESET, 0, &flag_res); if (err) { dev_err(hba->dev, "%s reading fHpbReset flag failed with error %d\n", __func__, err); return flag_res; } if (!flag_res) goto out; usleep_range(1000, 1100); } if (flag_res) { dev_err(hba->dev, "%s fHpbReset was not cleared by the device\n", __func__); } out: return flag_res; } void ufshpb_reset(struct ufs_hba *hba) { struct ufshpb_lu *hpb; struct scsi_device *sdev; shost_for_each_device(sdev, hba->host) { hpb = ufshpb_get_hpb_data(sdev); if (!hpb) continue; if (ufshpb_get_state(hpb) != HPB_RESET) continue; ufshpb_set_state(hpb, HPB_PRESENT); } } void ufshpb_reset_host(struct ufs_hba *hba) { struct ufshpb_lu *hpb; struct scsi_device *sdev; shost_for_each_device(sdev, hba->host) { hpb = ufshpb_get_hpb_data(sdev); if (!hpb) continue; if (ufshpb_get_state(hpb) != HPB_PRESENT) continue; ufshpb_set_state(hpb, HPB_RESET); ufshpb_cancel_jobs(hpb); ufshpb_discard_rsp_lists(hpb); } } void ufshpb_suspend(struct ufs_hba *hba) { struct ufshpb_lu *hpb; struct scsi_device *sdev; shost_for_each_device(sdev, hba->host) { hpb = ufshpb_get_hpb_data(sdev); if (!hpb) continue; if (ufshpb_get_state(hpb) != HPB_PRESENT) continue; ufshpb_set_state(hpb, HPB_SUSPEND); ufshpb_cancel_jobs(hpb); } } void ufshpb_resume(struct ufs_hba *hba) { struct ufshpb_lu *hpb; struct scsi_device *sdev; shost_for_each_device(sdev, hba->host) { hpb = ufshpb_get_hpb_data(sdev); if (!hpb) continue; if ((ufshpb_get_state(hpb) != HPB_PRESENT) && (ufshpb_get_state(hpb) != HPB_SUSPEND)) continue; ufshpb_set_state(hpb, HPB_PRESENT); ufshpb_kick_map_work(hpb); if (hpb->is_hcm) { unsigned int poll = hpb->params.timeout_polling_interval_ms; schedule_delayed_work(&hpb->ufshpb_read_to_work, msecs_to_jiffies(poll)); } } } static int ufshpb_get_lu_info(struct ufs_hba *hba, int lun, struct ufshpb_lu_info *hpb_lu_info) { u16 max_active_rgns; u8 lu_enable; int size; int ret; char desc_buf[QUERY_DESC_MAX_SIZE]; ufshcd_map_desc_id_to_length(hba, QUERY_DESC_IDN_UNIT, &size); pm_runtime_get_sync(hba->dev); ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, QUERY_DESC_IDN_UNIT, lun, 0, desc_buf, &size); pm_runtime_put_sync(hba->dev); if (ret) { dev_err(hba->dev, "%s: idn: %d lun: %d query request failed", __func__, QUERY_DESC_IDN_UNIT, lun); return ret; } lu_enable = desc_buf[UNIT_DESC_PARAM_LU_ENABLE]; if (lu_enable != LU_ENABLED_HPB_FUNC) return -ENODEV; max_active_rgns = get_unaligned_be16( desc_buf + UNIT_DESC_PARAM_HPB_LU_MAX_ACTIVE_RGNS); if (!max_active_rgns) { dev_err(hba->dev, "lun %d wrong number of max active regions\n", lun); return -ENODEV; } hpb_lu_info->num_blocks = get_unaligned_be64( desc_buf + UNIT_DESC_PARAM_LOGICAL_BLK_COUNT); hpb_lu_info->pinned_start = get_unaligned_be16( desc_buf + UNIT_DESC_PARAM_HPB_PIN_RGN_START_OFF); hpb_lu_info->num_pinned = get_unaligned_be16( desc_buf + UNIT_DESC_PARAM_HPB_NUM_PIN_RGNS); hpb_lu_info->max_active_rgns = max_active_rgns; return 0; } void ufshpb_destroy_lu(struct ufs_hba *hba, struct scsi_device *sdev) { struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); if (!hpb) return; ufshpb_set_state(hpb, HPB_FAILED); sdev = hpb->sdev_ufs_lu; sdev->hostdata = NULL; ufshpb_cancel_jobs(hpb); ufshpb_pre_req_mempool_destroy(hpb); ufshpb_destroy_region_tbl(hpb); kmem_cache_destroy(hpb->map_req_cache); kmem_cache_destroy(hpb->m_page_cache); list_del_init(&hpb->list_hpb_lu); kfree(hpb); } static void ufshpb_hpb_lu_prepared(struct ufs_hba *hba) { int pool_size; struct ufshpb_lu *hpb; struct scsi_device *sdev; bool init_success; if (tot_active_srgn_pages == 0) { ufshpb_remove(hba); return; } init_success = !ufshpb_check_hpb_reset_query(hba); pool_size = PAGE_ALIGN(ufshpb_host_map_kbytes * 1024) / PAGE_SIZE; if (pool_size > tot_active_srgn_pages) { mempool_resize(ufshpb_mctx_pool, tot_active_srgn_pages); mempool_resize(ufshpb_page_pool, tot_active_srgn_pages); } shost_for_each_device(sdev, hba->host) { hpb = ufshpb_get_hpb_data(sdev); if (!hpb) continue; if (init_success) { ufshpb_set_state(hpb, HPB_PRESENT); if ((hpb->lu_pinned_end - hpb->lu_pinned_start) > 0) queue_work(ufshpb_wq, &hpb->map_work); if (!hpb->is_hcm) ufshpb_issue_umap_all_req(hpb); } else { dev_err(hba->dev, "destroy HPB lu %d\n", hpb->lun); ufshpb_destroy_lu(hba, sdev); } } if (!init_success) ufshpb_remove(hba); } void ufshpb_init_hpb_lu(struct ufs_hba *hba, struct scsi_device *sdev) { struct ufshpb_lu *hpb; int ret; struct ufshpb_lu_info hpb_lu_info = { 0 }; int lun = sdev->lun; if (lun >= hba->dev_info.max_lu_supported) goto out; ret = ufshpb_get_lu_info(hba, lun, &hpb_lu_info); if (ret) goto out; hpb = ufshpb_alloc_hpb_lu(hba, sdev, &hba->ufshpb_dev, &hpb_lu_info); if (!hpb) goto out; tot_active_srgn_pages += hpb_lu_info.max_active_rgns * hpb->srgns_per_rgn * hpb->pages_per_srgn; out: /* All LUs are initialized */ if (atomic_dec_and_test(&hba->ufshpb_dev.slave_conf_cnt)) ufshpb_hpb_lu_prepared(hba); } static int ufshpb_init_mem_wq(struct ufs_hba *hba) { int ret; unsigned int pool_size; ufshpb_mctx_cache = kmem_cache_create("ufshpb_mctx_cache", sizeof(struct ufshpb_map_ctx), 0, 0, NULL); if (!ufshpb_mctx_cache) { dev_err(hba->dev, "ufshpb: cannot init mctx cache\n"); return -ENOMEM; } pool_size = PAGE_ALIGN(ufshpb_host_map_kbytes * 1024) / PAGE_SIZE; dev_info(hba->dev, "%s:%d ufshpb_host_map_kbytes %u pool_size %u\n", __func__, __LINE__, ufshpb_host_map_kbytes, pool_size); ufshpb_mctx_pool = mempool_create_slab_pool(pool_size, ufshpb_mctx_cache); if (!ufshpb_mctx_pool) { dev_err(hba->dev, "ufshpb: cannot init mctx pool\n"); ret = -ENOMEM; goto release_mctx_cache; } ufshpb_page_pool = mempool_create_page_pool(pool_size, 0); if (!ufshpb_page_pool) { dev_err(hba->dev, "ufshpb: cannot init page pool\n"); ret = -ENOMEM; goto release_mctx_pool; } ufshpb_wq = alloc_workqueue("ufshpb-wq", WQ_UNBOUND | WQ_MEM_RECLAIM, 0); if (!ufshpb_wq) { dev_err(hba->dev, "ufshpb: alloc workqueue failed\n"); ret = -ENOMEM; goto release_page_pool; } return 0; release_page_pool: mempool_destroy(ufshpb_page_pool); release_mctx_pool: mempool_destroy(ufshpb_mctx_pool); release_mctx_cache: kmem_cache_destroy(ufshpb_mctx_cache); return ret; } void ufshpb_get_geo_info(struct ufs_hba *hba, u8 *geo_buf) { struct ufshpb_dev_info *hpb_info = &hba->ufshpb_dev; int max_active_rgns = 0; int hpb_num_lu; hpb_num_lu = geo_buf[GEOMETRY_DESC_PARAM_HPB_NUMBER_LU]; if (hpb_num_lu == 0) { dev_err(hba->dev, "No HPB LU supported\n"); hpb_info->hpb_disabled = true; return; } hpb_info->rgn_size = geo_buf[GEOMETRY_DESC_PARAM_HPB_REGION_SIZE]; hpb_info->srgn_size = geo_buf[GEOMETRY_DESC_PARAM_HPB_SUBREGION_SIZE]; max_active_rgns = get_unaligned_be16(geo_buf + GEOMETRY_DESC_PARAM_HPB_MAX_ACTIVE_REGS); if (hpb_info->rgn_size == 0 || hpb_info->srgn_size == 0 || max_active_rgns == 0) { dev_err(hba->dev, "No HPB supported device\n"); hpb_info->hpb_disabled = true; return; } } void ufshpb_get_dev_info(struct ufs_hba *hba, u8 *desc_buf) { struct ufshpb_dev_info *hpb_dev_info = &hba->ufshpb_dev; int version, ret; u32 max_hpb_single_cmd = HPB_MULTI_CHUNK_LOW; hpb_dev_info->control_mode = desc_buf[DEVICE_DESC_PARAM_HPB_CONTROL]; version = get_unaligned_be16(desc_buf + DEVICE_DESC_PARAM_HPB_VER); if ((version != HPB_SUPPORT_VERSION) && (version != HPB_SUPPORT_LEGACY_VERSION)) { dev_err(hba->dev, "%s: HPB %x version is not supported.\n", __func__, version); hpb_dev_info->hpb_disabled = true; return; } if (version == HPB_SUPPORT_LEGACY_VERSION) hpb_dev_info->is_legacy = true; pm_runtime_get_sync(hba->dev); ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, QUERY_ATTR_IDN_MAX_HPB_SINGLE_CMD, 0, 0, &max_hpb_single_cmd); pm_runtime_put_sync(hba->dev); if (ret) dev_err(hba->dev, "%s: idn: read max size of single hpb cmd query request failed", __func__); hpb_dev_info->max_hpb_single_cmd = max_hpb_single_cmd; /* * Get the number of user logical unit to check whether all * scsi_device finish initialization */ hpb_dev_info->num_lu = desc_buf[DEVICE_DESC_PARAM_NUM_LU]; } void ufshpb_init(struct ufs_hba *hba) { struct ufshpb_dev_info *hpb_dev_info = &hba->ufshpb_dev; int try; int ret; if (!ufshpb_is_allowed(hba) || !hba->dev_info.hpb_enabled) return; if (ufshpb_init_mem_wq(hba)) { hpb_dev_info->hpb_disabled = true; return; } atomic_set(&hpb_dev_info->slave_conf_cnt, hpb_dev_info->num_lu); tot_active_srgn_pages = 0; /* issue HPB reset query */ for (try = 0; try < HPB_RESET_REQ_RETRIES; try++) { ret = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG, QUERY_FLAG_IDN_HPB_RESET, 0, NULL); if (!ret) break; } } void ufshpb_remove(struct ufs_hba *hba) { mempool_destroy(ufshpb_page_pool); mempool_destroy(ufshpb_mctx_pool); kmem_cache_destroy(ufshpb_mctx_cache); destroy_workqueue(ufshpb_wq); } module_param(ufshpb_host_map_kbytes, uint, 0644); MODULE_PARM_DESC(ufshpb_host_map_kbytes, "ufshpb host mapping memory kilo-bytes for ufshpb memory-pool");
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1