From 2777e73fc154e2e87233bdcc0e2402b33815198e Mon Sep 17 00:00:00 2001 From: Maramaina Naresh Date: Tue, 19 Dec 2023 18:07:05 +0530 Subject: scsi: ufs: core: Add CPU latency QoS support for UFS driver Register UFS driver to CPU latency PM QoS framework to improve UFS device random I/O performance. PM QoS initialization will insert new QoS request into the CPU latency QoS list with the maximum latency PM_QOS_DEFAULT_VALUE value. The UFS driver will vote for performance mode on scale up and power save mode for scale down. If clock scaling feature is not enabled then voting will be based on clock on or off condition. Also provide a sysfs interface to enable/disable PM QoS feature. tiotest benchmark tool I/O performance results on sm8550 platform: 1. Without PM QoS support Type (Speed in) | Average of 18 iterations Random Write(IPOS) | 41065.13 Random Read(IPOS) | 37101.3 2. With PM QoS support Type (Speed in) | Average of 18 iterations Random Write(IPOS) | 46784.9 Random Read(IPOS) | 42943.4 (Improvement with PM QoS = ~15%). Reviewed-by: Peter Wang Reviewed-by: AngeloGioacchino Del Regno Co-developed-by: Nitin Rawat Signed-off-by: Nitin Rawat Co-developed-by: Naveen Kumar Goud Arepalli Signed-off-by: Naveen Kumar Goud Arepalli Signed-off-by: Maramaina Naresh Link: https://lore.kernel.org/r/20231219123706.6463-2-quic_mnaresh@quicinc.com Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufs-sysfs.c | 49 +++++++++++++++++++++++++++++++++++++++++++ drivers/ufs/core/ufshcd.c | 50 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) (limited to 'drivers/ufs/core') diff --git a/drivers/ufs/core/ufs-sysfs.c b/drivers/ufs/core/ufs-sysfs.c index e6d12289e017..3d049967f6bc 100644 --- a/drivers/ufs/core/ufs-sysfs.c +++ b/drivers/ufs/core/ufs-sysfs.c @@ -405,6 +405,53 @@ static ssize_t wb_flush_threshold_store(struct device *dev, return count; } +/** + * pm_qos_enable_show - sysfs handler to show pm qos enable value + * @dev: device associated with the UFS controller + * @attr: sysfs attribute handle + * @buf: buffer for sysfs file + * + * Print 1 if PM QoS feature is enabled, 0 if disabled. + * + * Returns number of characters written to @buf. + */ +static ssize_t pm_qos_enable_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + return sysfs_emit(buf, "%d\n", hba->pm_qos_enabled); +} + +/** + * pm_qos_enable_store - sysfs handler to store value + * @dev: device associated with the UFS controller + * @attr: sysfs attribute handle + * @buf: buffer for sysfs file + * @count: stores buffer characters count + * + * Input 0 to disable PM QoS and 1 value to enable. + * Default state: 1 + * + * Return: number of characters written to @buf on success, < 0 upon failure. + */ +static ssize_t pm_qos_enable_store(struct device *dev, + struct device_attribute *attr, const char *buf, size_t count) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + bool value; + + if (kstrtobool(buf, &value)) + return -EINVAL; + + if (value) + ufshcd_pm_qos_init(hba); + else + ufshcd_pm_qos_exit(hba); + + return count; +} + static DEVICE_ATTR_RW(rpm_lvl); static DEVICE_ATTR_RO(rpm_target_dev_state); static DEVICE_ATTR_RO(rpm_target_link_state); @@ -416,6 +463,7 @@ static DEVICE_ATTR_RW(wb_on); static DEVICE_ATTR_RW(enable_wb_buf_flush); static DEVICE_ATTR_RW(wb_flush_threshold); static DEVICE_ATTR_RW(rtc_update_ms); +static DEVICE_ATTR_RW(pm_qos_enable); static struct attribute *ufs_sysfs_ufshcd_attrs[] = { &dev_attr_rpm_lvl.attr, @@ -429,6 +477,7 @@ static struct attribute *ufs_sysfs_ufshcd_attrs[] = { &dev_attr_enable_wb_buf_flush.attr, &dev_attr_wb_flush_threshold.attr, &dev_attr_rtc_update_ms.attr, + &dev_attr_pm_qos_enable.attr, NULL }; diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 029d017fc1b6..10b0c9cea0f0 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -1014,6 +1014,48 @@ static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba) return ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6; } +/** + * ufshcd_pm_qos_init - initialize PM QoS request + * @hba: per adapter instance + */ +void ufshcd_pm_qos_init(struct ufs_hba *hba) +{ + + if (hba->pm_qos_enabled) + return; + + cpu_latency_qos_add_request(&hba->pm_qos_req, PM_QOS_DEFAULT_VALUE); + + if (cpu_latency_qos_request_active(&hba->pm_qos_req)) + hba->pm_qos_enabled = true; +} + +/** + * ufshcd_pm_qos_exit - remove request from PM QoS + * @hba: per adapter instance + */ +void ufshcd_pm_qos_exit(struct ufs_hba *hba) +{ + if (!hba->pm_qos_enabled) + return; + + cpu_latency_qos_remove_request(&hba->pm_qos_req); + hba->pm_qos_enabled = false; +} + +/** + * ufshcd_pm_qos_update - update PM QoS request + * @hba: per adapter instance + * @on: If True, vote for perf PM QoS mode otherwise power save mode + */ +static void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on) +{ + if (!hba->pm_qos_enabled) + return; + + cpu_latency_qos_update_request(&hba->pm_qos_req, on ? 0 : PM_QOS_DEFAULT_VALUE); +} + /** * ufshcd_set_clk_freq - set UFS controller clock frequencies * @hba: per adapter instance @@ -1160,8 +1202,11 @@ static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, hba->devfreq->previous_freq); else ufshcd_set_clk_freq(hba, !scale_up); + goto out; } + ufshcd_pm_qos_update(hba, scale_up); + out: trace_ufshcd_profile_clk_scaling(dev_name(hba->dev), (scale_up ? "up" : "down"), @@ -9279,6 +9324,8 @@ static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) if (ret) return ret; + if (!ufshcd_is_clkscaling_supported(hba)) + ufshcd_pm_qos_update(hba, on); out: if (ret) { list_for_each_entry(clki, head, list) { @@ -9456,6 +9503,7 @@ out: static void ufshcd_hba_exit(struct ufs_hba *hba) { if (hba->is_powered) { + ufshcd_pm_qos_exit(hba); ufshcd_exit_clk_scaling(hba); ufshcd_exit_clk_gating(hba); if (hba->eh_wq) @@ -10108,6 +10156,7 @@ static int ufshcd_suspend(struct ufs_hba *hba) ufshcd_vreg_set_lpm(hba); /* Put the host controller in low power mode if possible */ ufshcd_hba_vreg_set_lpm(hba); + ufshcd_pm_qos_update(hba, false); return ret; } @@ -10654,6 +10703,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) ufs_sysfs_add_nodes(hba->dev); device_enable_async_suspend(dev); + ufshcd_pm_qos_init(hba); return 0; free_tmf_queue: -- cgit v1.2.3 From 4380e64a94e16c757552e8e2fbdc856415012fc8 Mon Sep 17 00:00:00 2001 From: Peter Wang Date: Tue, 9 Jan 2024 20:40:14 +0800 Subject: scsi: core: Move autosuspend timer delay to Scsi_Host The runtime suspend timer delay is a const value in scsi_host_template which a host driver cannot modify at runtime. Move the delay to Scsi_Host to allow a driver to update it. Signed-off-by: Peter Wang Link: https://lore.kernel.org/r/20240109124015.31359-2-peter.wang@mediatek.com Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- drivers/scsi/sd.c | 2 +- drivers/ufs/core/ufshcd.c | 9 +++++++-- include/scsi/scsi_host.h | 6 +++--- 3 files changed, 11 insertions(+), 6 deletions(-) (limited to 'drivers/ufs/core') diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 0833b3e6aa6e..d1b87670764c 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -3728,7 +3728,7 @@ static int sd_probe(struct device *dev) blk_pm_runtime_init(sdp->request_queue, dev); if (sdp->rpm_autosuspend) { pm_runtime_set_autosuspend_delay(dev, - sdp->host->hostt->rpm_autosuspend_delay); + sdp->host->rpm_autosuspend_delay); } error = device_add_disk(dev, gd, NULL); diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 10b0c9cea0f0..e519695b704f 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -8031,11 +8031,13 @@ out: static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev) { + struct Scsi_Host *shost = sdev->host; + scsi_autopm_get_device(sdev); blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev); if (sdev->rpm_autosuspend) pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev, - RPM_AUTOSUSPEND_DELAY_MS); + shost->rpm_autosuspend_delay); scsi_autopm_put_device(sdev); } @@ -9109,7 +9111,6 @@ static const struct scsi_host_template ufshcd_driver_template = { .track_queue_depth = 1, .skip_settle_delay = 1, .sdev_groups = ufshcd_driver_groups, - .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS, }; static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg, @@ -10568,6 +10569,10 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) host->max_cmd_len = UFS_CDB_SIZE; host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING); + /* Use default RPM delay if host not set */ + if (host->rpm_autosuspend_delay == 0) + host->rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS; + hba->max_pwr_info.is_valid = false; /* Initialize work queues */ diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h index 3b907fc2ef08..b259d42a1e1a 100644 --- a/include/scsi/scsi_host.h +++ b/include/scsi/scsi_host.h @@ -497,9 +497,6 @@ struct scsi_host_template { * scsi_netlink.h */ u64 vendor_id; - - /* Delay for runtime autosuspend */ - int rpm_autosuspend_delay; }; /* @@ -713,6 +710,9 @@ struct Scsi_Host { */ struct device *dma_dev; + /* Delay for runtime autosuspend */ + int rpm_autosuspend_delay; + /* * We should ensure that this is aligned, both for better performance * and also because some compilers (m68k) don't automatically force -- cgit v1.2.3 From ab3e6c4e0ea149f16d5b719ecf7572862060d215 Mon Sep 17 00:00:00 2001 From: ChanWoo Lee Date: Tue, 2 Jan 2024 10:42:22 +0900 Subject: scsi: ufs: mcq: Add definition for REG_UFS_MEM_CFG register Instead of hardcoding the register field, add the proper definition. While at it, let's also use ufshcd_rmwl() to simplify updating this register. Reviewed-by: Peter Wang Signed-off-by: ChanWoo Lee Link: https://lore.kernel.org/r/20240102014222.23351-1-cw9316.lee@samsung.com Reviewed-by: Manivannan Sadhasivam Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufs-mcq.c | 6 ++++++ drivers/ufs/core/ufshcd.c | 4 +--- drivers/ufs/host/ufs-mediatek.c | 4 +--- include/ufs/ufshcd.h | 1 + include/ufs/ufshci.h | 3 +++ 5 files changed, 12 insertions(+), 6 deletions(-) (limited to 'drivers/ufs/core') diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index 0787456c2b89..edc752e55878 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -399,6 +399,12 @@ void ufshcd_mcq_enable_esi(struct ufs_hba *hba) } EXPORT_SYMBOL_GPL(ufshcd_mcq_enable_esi); +void ufshcd_mcq_enable(struct ufs_hba *hba) +{ + ufshcd_rmwl(hba, MCQ_MODE_SELECT, MCQ_MODE_SELECT, REG_UFS_MEM_CFG); +} +EXPORT_SYMBOL_GPL(ufshcd_mcq_enable); + void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg) { ufshcd_writel(hba, msg->address_lo, REG_UFS_ESILBA); diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index e519695b704f..f10a92681bba 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -8847,9 +8847,7 @@ static void ufshcd_config_mcq(struct ufs_hba *hba) hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; hba->reserved_slot = hba->nutrs - UFSHCD_NUM_RESERVED; - /* Select MCQ mode */ - ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x1, - REG_UFS_MEM_CFG); + ufshcd_mcq_enable(hba); hba->mcq_enabled = true; dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n", diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c index 72a7b3a7cc00..b8a8801322e2 100644 --- a/drivers/ufs/host/ufs-mediatek.c +++ b/drivers/ufs/host/ufs-mediatek.c @@ -1254,9 +1254,7 @@ static int ufs_mtk_link_set_hpm(struct ufs_hba *hba) ufs_mtk_config_mcq(hba, false); ufshcd_mcq_make_queues_operational(hba); ufshcd_mcq_config_mac(hba, hba->nutrs); - /* Enable MCQ mode */ - ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x1, - REG_UFS_MEM_CFG); + ufshcd_mcq_enable(hba); } return 0; diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h index c491671e79b7..cb2afcebbdf5 100644 --- a/include/ufs/ufshcd.h +++ b/include/ufs/ufshcd.h @@ -1267,6 +1267,7 @@ unsigned long ufshcd_mcq_poll_cqe_lock(struct ufs_hba *hba, struct ufs_hw_queue *hwq); void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba); void ufshcd_mcq_enable_esi(struct ufs_hba *hba); +void ufshcd_mcq_enable(struct ufs_hba *hba); void ufshcd_mcq_config_esi(struct ufs_hba *hba, struct msi_msg *msg); int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table, diff --git a/include/ufs/ufshci.h b/include/ufs/ufshci.h index d5accacae6bc..a196e1c4c3bb 100644 --- a/include/ufs/ufshci.h +++ b/include/ufs/ufshci.h @@ -282,6 +282,9 @@ enum { /* UTMRLRSR - UTP Task Management Request Run-Stop Register 80h */ #define UTP_TASK_REQ_LIST_RUN_STOP_BIT 0x1 +/* REG_UFS_MEM_CFG - Global Config Registers 300h */ +#define MCQ_MODE_SELECT BIT(0) + /* CQISy - CQ y Interrupt Status Register */ #define UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS 0x1 -- cgit v1.2.3 From 325ec4ac7da6272da9b2da51b7c5cc75e48bf654 Mon Sep 17 00:00:00 2001 From: ChanWoo Lee Date: Fri, 5 Jan 2024 11:10:40 +0900 Subject: scsi: ufs: mcq: Use ufshcd_mcq_req_to_hwq() to simplify updating hwq Use ufshcd_mcq_req_to_hwq() to remove unnecessary variables and simplify. Signed-off-by: ChanWoo Lee Link: https://lore.kernel.org/r/20240105021041.20400-2-cw9316.lee@samsung.com Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'drivers/ufs/core') diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index f10a92681bba..c416826762e9 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -5645,7 +5645,6 @@ static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, struct ufshcd_lrb *lrbp; struct scsi_cmnd *cmd; unsigned long flags; - u32 hwq_num, utag; int tag; for (tag = 0; tag < hba->nutrs; tag++) { @@ -5655,9 +5654,7 @@ static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, test_bit(SCMD_STATE_COMPLETE, &cmd->state)) continue; - utag = blk_mq_unique_tag(scsi_cmd_to_rq(cmd)); - hwq_num = blk_mq_unique_tag_to_hwq(utag); - hwq = &hba->uhq[hwq_num]; + hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); if (force_compl) { ufshcd_mcq_compl_all_cqes_lock(hba, hwq); -- cgit v1.2.3 From 01f256228c0f89c4b48fbc7c67b64a26cdcfd740 Mon Sep 17 00:00:00 2001 From: ChanWoo Lee Date: Fri, 5 Jan 2024 11:10:41 +0900 Subject: scsi: ufs: mcq: Remove unused parameters The 'hwq' parameter is not used in this function. Remove unused parameters. Signed-off-by: ChanWoo Lee Link: https://lore.kernel.org/r/20240105021041.20400-3-cw9316.lee@samsung.com Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufs-mcq.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'drivers/ufs/core') diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index edc752e55878..8db81f1a12d5 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -258,9 +258,7 @@ EXPORT_SYMBOL_GPL(ufshcd_mcq_write_cqis); * Current MCQ specification doesn't provide a Task Tag or its equivalent in * the Completion Queue Entry. Find the Task Tag using an indirect method. */ -static int ufshcd_mcq_get_tag(struct ufs_hba *hba, - struct ufs_hw_queue *hwq, - struct cq_entry *cqe) +static int ufshcd_mcq_get_tag(struct ufs_hba *hba, struct cq_entry *cqe) { u64 addr; @@ -278,7 +276,7 @@ static void ufshcd_mcq_process_cqe(struct ufs_hba *hba, struct ufs_hw_queue *hwq) { struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq); - int tag = ufshcd_mcq_get_tag(hba, hwq, cqe); + int tag = ufshcd_mcq_get_tag(hba, cqe); if (cqe->command_desc_base_addr) { ufshcd_compl_one_cqe(hba, tag, cqe); -- cgit v1.2.3 From b8c3a7bac9b6cddeeb7cb82f3372310ecabf83ef Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Mon, 22 Jan 2024 18:22:19 -0600 Subject: scsi: ufs: Have midlayer retry start stop errors This has the SCSI midlayer retry errors instead of driving them itself. Signed-off-by: Mike Christie Link: https://lore.kernel.org/r/20240123002220.129141-19-michael.christie@oracle.com Acked-by: Christoph Hellwig Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'drivers/ufs/core') diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 029d017fc1b6..5cd9509e0c31 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -9475,7 +9475,17 @@ static int ufshcd_execute_start_stop(struct scsi_device *sdev, struct scsi_sense_hdr *sshdr) { const unsigned char cdb[6] = { START_STOP, 0, 0, 0, pwr_mode << 4, 0 }; + struct scsi_failure failure_defs[] = { + { + .allowed = 2, + .result = SCMD_FAILURE_RESULT_ANY, + }, + }; + struct scsi_failures failures = { + .failure_definitions = failure_defs, + }; const struct scsi_exec_args args = { + .failures = &failures, .sshdr = sshdr, .req_flags = BLK_MQ_REQ_PM, .scmd_flags = SCMD_FAIL_IF_RECOVERING, @@ -9501,7 +9511,7 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, struct scsi_sense_hdr sshdr; struct scsi_device *sdp; unsigned long flags; - int ret, retries; + int ret; spin_lock_irqsave(hba->host->host_lock, flags); sdp = hba->ufs_device_wlun; @@ -9527,15 +9537,7 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, * callbacks hence set the RQF_PM flag so that it doesn't resume the * already suspended childs. */ - for (retries = 3; retries > 0; --retries) { - ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr); - /* - * scsi_execute() only returns a negative value if the request - * queue is dying. - */ - if (ret <= 0) - break; - } + ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr); if (ret) { sdev_printk(KERN_WARNING, sdp, "START_STOP failed for power mode: %d, result %x\n", -- cgit v1.2.3