diff options
Diffstat (limited to 'drivers/ufs')
-rw-r--r-- | drivers/ufs/core/Kconfig | 8 | ||||
-rw-r--r-- | drivers/ufs/core/Makefile | 1 | ||||
-rw-r--r-- | drivers/ufs/core/ufs-hwmon.c | 3 | ||||
-rw-r--r-- | drivers/ufs/core/ufs-mcq.c | 34 | ||||
-rw-r--r-- | drivers/ufs/core/ufs-sysfs.c | 22 | ||||
-rw-r--r-- | drivers/ufs/core/ufs_bsg.c | 2 | ||||
-rw-r--r-- | drivers/ufs/core/ufshcd-crypto.h | 20 | ||||
-rw-r--r-- | drivers/ufs/core/ufshcd-priv.h | 4 | ||||
-rw-r--r-- | drivers/ufs/core/ufshcd.c | 627 | ||||
-rw-r--r-- | drivers/ufs/core/ufshpb.c | 2668 | ||||
-rw-r--r-- | drivers/ufs/core/ufshpb.h | 318 | ||||
-rw-r--r-- | drivers/ufs/host/cdns-pltfrm.c | 27 | ||||
-rw-r--r-- | drivers/ufs/host/tc-dwc-g210-pci.c | 2 | ||||
-rw-r--r-- | drivers/ufs/host/tc-dwc-g210.c | 32 | ||||
-rw-r--r-- | drivers/ufs/host/ti-j721e-ufs.c | 2 | ||||
-rw-r--r-- | drivers/ufs/host/ufs-mediatek.c | 180 | ||||
-rw-r--r-- | drivers/ufs/host/ufs-mediatek.h | 33 | ||||
-rw-r--r-- | drivers/ufs/host/ufs-qcom.c | 174 | ||||
-rw-r--r-- | drivers/ufs/host/ufs-qcom.h | 4 | ||||
-rw-r--r-- | drivers/ufs/host/ufs-renesas.c | 2 | ||||
-rw-r--r-- | drivers/ufs/host/ufshcd-dwc.c | 22 | ||||
-rw-r--r-- | drivers/ufs/host/ufshcd-pci.c | 3 | ||||
-rw-r--r-- | drivers/ufs/host/ufshcd-pltfrm.c | 13 |
23 files changed, 746 insertions, 3455 deletions
diff --git a/drivers/ufs/core/Kconfig b/drivers/ufs/core/Kconfig index e11978171403..817208ee64ec 100644 --- a/drivers/ufs/core/Kconfig +++ b/drivers/ufs/core/Kconfig @@ -35,14 +35,6 @@ config SCSI_UFS_CRYPTO capabilities of the UFS device (if present) to perform crypto operations on data being transferred to/from the device. -config SCSI_UFS_HPB - bool "Support UFS Host Performance Booster" - help - The UFS HPB feature improves random read performance. It caches - L2P (logical to physical) map of UFS to host DRAM. The driver uses HPB - read command by piggybacking physical page number for bypassing FTL (flash - translation layer)'s L2P address translation. - config SCSI_UFS_FAULT_INJECTION bool "UFS Fault Injection Support" depends on FAULT_INJECTION diff --git a/drivers/ufs/core/Makefile b/drivers/ufs/core/Makefile index 4d02e0f2de10..cf820fa09a04 100644 --- a/drivers/ufs/core/Makefile +++ b/drivers/ufs/core/Makefile @@ -5,6 +5,5 @@ ufshcd-core-y += ufshcd.o ufs-sysfs.o ufs-mcq.o ufshcd-core-$(CONFIG_DEBUG_FS) += ufs-debugfs.o ufshcd-core-$(CONFIG_SCSI_UFS_BSG) += ufs_bsg.o ufshcd-core-$(CONFIG_SCSI_UFS_CRYPTO) += ufshcd-crypto.o -ufshcd-core-$(CONFIG_SCSI_UFS_HPB) += ufshpb.o ufshcd-core-$(CONFIG_SCSI_UFS_FAULT_INJECTION) += ufs-fault-injection.o ufshcd-core-$(CONFIG_SCSI_UFS_HWMON) += ufs-hwmon.o diff --git a/drivers/ufs/core/ufs-hwmon.c b/drivers/ufs/core/ufs-hwmon.c index 101d7082446f..34194064367f 100644 --- a/drivers/ufs/core/ufs-hwmon.c +++ b/drivers/ufs/core/ufs-hwmon.c @@ -127,7 +127,8 @@ static int ufs_hwmon_write(struct device *dev, enum hwmon_sensor_types type, u32 return err; } -static umode_t ufs_hwmon_is_visible(const void *_data, enum hwmon_sensor_types type, u32 attr, +static umode_t ufs_hwmon_is_visible(const void *data, + enum hwmon_sensor_types type, u32 attr, int channel) { if (type != hwmon_temp) diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index 386674ead7f0..2ba8ec254dce 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -97,6 +97,7 @@ void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds) val |= FIELD_PREP(MCQ_CFG_MAC_MASK, max_active_cmds); ufshcd_writel(hba, val, REG_UFS_MCQ_CFG); } +EXPORT_SYMBOL_GPL(ufshcd_mcq_config_mac); /** * ufshcd_mcq_req_to_hwq - find the hardware queue on which the @@ -104,7 +105,7 @@ void ufshcd_mcq_config_mac(struct ufs_hba *hba, u32 max_active_cmds) * @hba: per adapter instance * @req: pointer to the request to be issued * - * Returns the hardware queue instance on which the request would + * Return: the hardware queue instance on which the request would * be queued. */ struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba, @@ -120,7 +121,7 @@ struct ufs_hw_queue *ufshcd_mcq_req_to_hwq(struct ufs_hba *hba, * ufshcd_mcq_decide_queue_depth - decide the queue depth * @hba: per adapter instance * - * Returns queue-depth on success, non-zero on error + * Return: queue-depth on success, non-zero on error * * MAC - Max. Active Command of the Host Controller (HC) * HC wouldn't send more than this commands to the device. @@ -245,6 +246,7 @@ u32 ufshcd_mcq_read_cqis(struct ufs_hba *hba, int i) { return readl(mcq_opr_base(hba, OPR_CQIS, i) + REG_CQIS); } +EXPORT_SYMBOL_GPL(ufshcd_mcq_read_cqis); void ufshcd_mcq_write_cqis(struct ufs_hba *hba, u32 val, int i) { @@ -388,6 +390,7 @@ void ufshcd_mcq_make_queues_operational(struct ufs_hba *hba) MCQ_CFG_n(REG_SQATTR, i)); } } +EXPORT_SYMBOL_GPL(ufshcd_mcq_make_queues_operational); void ufshcd_mcq_enable_esi(struct ufs_hba *hba) { @@ -487,10 +490,10 @@ static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq) /** * ufshcd_mcq_sq_cleanup - Clean up submission queue resources * associated with the pending command. - * @hba - per adapter instance. - * @task_tag - The command's task tag. + * @hba: per adapter instance. + * @task_tag: The command's task tag. * - * Returns 0 for success; error code otherwise. + * Return: 0 for success; error code otherwise. */ int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag) { @@ -551,16 +554,11 @@ unlock: * Write the sqe's Command Type to 0xF. The host controller will not * fetch any sqe with Command Type = 0xF. * - * @utrd - UTP Transfer Request Descriptor to be nullified. + * @utrd: UTP Transfer Request Descriptor to be nullified. */ static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd) { - u32 dword_0; - - dword_0 = le32_to_cpu(utrd->header.dword_0); - dword_0 &= ~UPIU_COMMAND_TYPE_MASK; - dword_0 |= FIELD_PREP(UPIU_COMMAND_TYPE_MASK, 0xF); - utrd->header.dword_0 = cpu_to_le32(dword_0); + utrd->header.command_type = 0xf; } /** @@ -568,11 +566,11 @@ static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd) * If the command is in the submission queue and not issued to the device yet, * nullify the sqe so the host controller will skip fetching the sqe. * - * @hba - per adapter instance. - * @hwq - Hardware Queue to be searched. - * @task_tag - The command's task tag. + * @hba: per adapter instance. + * @hwq: Hardware Queue to be searched. + * @task_tag: The command's task tag. * - * Returns true if the SQE containing the command is present in the SQ + * Return: true if the SQE containing the command is present in the SQ * (not fetched by the controller); returns false if the SQE is not in the SQ. */ static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba, @@ -621,9 +619,9 @@ out: /** * ufshcd_mcq_abort - Abort the command in MCQ. - * @cmd - The command to be aborted. + * @cmd: The command to be aborted. * - * Returns SUCCESS or FAILED error codes + * Return: SUCCESS or FAILED error codes */ int ufshcd_mcq_abort(struct scsi_cmnd *cmd) { diff --git a/drivers/ufs/core/ufs-sysfs.c b/drivers/ufs/core/ufs-sysfs.c index 6c72075750dd..c95906443d5f 100644 --- a/drivers/ufs/core/ufs-sysfs.c +++ b/drivers/ufs/core/ufs-sysfs.c @@ -718,8 +718,6 @@ UFS_DEVICE_DESC_PARAM(device_version, _DEV_VER, 2); UFS_DEVICE_DESC_PARAM(number_of_secure_wpa, _NUM_SEC_WPA, 1); UFS_DEVICE_DESC_PARAM(psa_max_data_size, _PSA_MAX_DATA, 4); UFS_DEVICE_DESC_PARAM(psa_state_timeout, _PSA_TMT, 1); -UFS_DEVICE_DESC_PARAM(hpb_version, _HPB_VER, 2); -UFS_DEVICE_DESC_PARAM(hpb_control, _HPB_CONTROL, 1); UFS_DEVICE_DESC_PARAM(ext_feature_sup, _EXT_UFS_FEATURE_SUP, 4); UFS_DEVICE_DESC_PARAM(wb_presv_us_en, _WB_PRESRV_USRSPC_EN, 1); UFS_DEVICE_DESC_PARAM(wb_type, _WB_TYPE, 1); @@ -752,8 +750,6 @@ static struct attribute *ufs_sysfs_device_descriptor[] = { &dev_attr_number_of_secure_wpa.attr, &dev_attr_psa_max_data_size.attr, &dev_attr_psa_state_timeout.attr, - &dev_attr_hpb_version.attr, - &dev_attr_hpb_control.attr, &dev_attr_ext_feature_sup.attr, &dev_attr_wb_presv_us_en.attr, &dev_attr_wb_type.attr, @@ -827,10 +823,6 @@ UFS_GEOMETRY_DESC_PARAM(enh4_memory_max_alloc_units, _ENM4_MAX_NUM_UNITS, 4); UFS_GEOMETRY_DESC_PARAM(enh4_memory_capacity_adjustment_factor, _ENM4_CAP_ADJ_FCTR, 2); -UFS_GEOMETRY_DESC_PARAM(hpb_region_size, _HPB_REGION_SIZE, 1); -UFS_GEOMETRY_DESC_PARAM(hpb_number_lu, _HPB_NUMBER_LU, 1); -UFS_GEOMETRY_DESC_PARAM(hpb_subregion_size, _HPB_SUBREGION_SIZE, 1); -UFS_GEOMETRY_DESC_PARAM(hpb_max_active_regions, _HPB_MAX_ACTIVE_REGS, 2); UFS_GEOMETRY_DESC_PARAM(wb_max_alloc_units, _WB_MAX_ALLOC_UNITS, 4); UFS_GEOMETRY_DESC_PARAM(wb_max_wb_luns, _WB_MAX_WB_LUNS, 1); UFS_GEOMETRY_DESC_PARAM(wb_buff_cap_adj, _WB_BUFF_CAP_ADJ, 1); @@ -868,10 +860,6 @@ static struct attribute *ufs_sysfs_geometry_descriptor[] = { &dev_attr_enh3_memory_capacity_adjustment_factor.attr, &dev_attr_enh4_memory_max_alloc_units.attr, &dev_attr_enh4_memory_capacity_adjustment_factor.attr, - &dev_attr_hpb_region_size.attr, - &dev_attr_hpb_number_lu.attr, - &dev_attr_hpb_subregion_size.attr, - &dev_attr_hpb_max_active_regions.attr, &dev_attr_wb_max_alloc_units.attr, &dev_attr_wb_max_wb_luns.attr, &dev_attr_wb_buff_cap_adj.attr, @@ -1132,7 +1120,6 @@ UFS_FLAG(disable_fw_update, _PERMANENTLY_DISABLE_FW_UPDATE); UFS_FLAG(wb_enable, _WB_EN); UFS_FLAG(wb_flush_en, _WB_BUFF_FLUSH_EN); UFS_FLAG(wb_flush_during_h8, _WB_BUFF_FLUSH_DURING_HIBERN8); -UFS_FLAG(hpb_enable, _HPB_EN); static struct attribute *ufs_sysfs_device_flags[] = { &dev_attr_device_init.attr, @@ -1146,7 +1133,6 @@ static struct attribute *ufs_sysfs_device_flags[] = { &dev_attr_wb_enable.attr, &dev_attr_wb_flush_en.attr, &dev_attr_wb_flush_during_h8.attr, - &dev_attr_hpb_enable.attr, NULL, }; @@ -1193,7 +1179,6 @@ out: \ static DEVICE_ATTR_RO(_name) UFS_ATTRIBUTE(boot_lun_enabled, _BOOT_LU_EN); -UFS_ATTRIBUTE(max_data_size_hpb_single_cmd, _MAX_HPB_SINGLE_CMD); UFS_ATTRIBUTE(current_power_mode, _POWER_MODE); UFS_ATTRIBUTE(active_icc_level, _ACTIVE_ICC_LVL); UFS_ATTRIBUTE(ooo_data_enabled, _OOO_DATA_EN); @@ -1217,7 +1202,6 @@ UFS_ATTRIBUTE(wb_cur_buf, _CURR_WB_BUFF_SIZE); static struct attribute *ufs_sysfs_attributes[] = { &dev_attr_boot_lun_enabled.attr, - &dev_attr_max_data_size_hpb_single_cmd.attr, &dev_attr_current_power_mode.attr, &dev_attr_active_icc_level.attr, &dev_attr_ooo_data_enabled.attr, @@ -1291,9 +1275,6 @@ UFS_UNIT_DESC_PARAM(provisioning_type, _PROVISIONING_TYPE, 1); UFS_UNIT_DESC_PARAM(physical_memory_resourse_count, _PHY_MEM_RSRC_CNT, 8); UFS_UNIT_DESC_PARAM(context_capabilities, _CTX_CAPABILITIES, 2); UFS_UNIT_DESC_PARAM(large_unit_granularity, _LARGE_UNIT_SIZE_M1, 1); -UFS_UNIT_DESC_PARAM(hpb_lu_max_active_regions, _HPB_LU_MAX_ACTIVE_RGNS, 2); -UFS_UNIT_DESC_PARAM(hpb_pinned_region_start_offset, _HPB_PIN_RGN_START_OFF, 2); -UFS_UNIT_DESC_PARAM(hpb_number_pinned_regions, _HPB_NUM_PIN_RGNS, 2); UFS_UNIT_DESC_PARAM(wb_buf_alloc_units, _WB_BUF_ALLOC_UNITS, 4); static struct attribute *ufs_sysfs_unit_descriptor[] = { @@ -1311,9 +1292,6 @@ static struct attribute *ufs_sysfs_unit_descriptor[] = { &dev_attr_physical_memory_resourse_count.attr, &dev_attr_context_capabilities.attr, &dev_attr_large_unit_granularity.attr, - &dev_attr_hpb_lu_max_active_regions.attr, - &dev_attr_hpb_pinned_region_start_offset.attr, - &dev_attr_hpb_number_pinned_regions.attr, &dev_attr_wb_buf_alloc_units.attr, NULL, }; diff --git a/drivers/ufs/core/ufs_bsg.c b/drivers/ufs/core/ufs_bsg.c index 0d38e7fa34cc..34e423924e06 100644 --- a/drivers/ufs/core/ufs_bsg.c +++ b/drivers/ufs/core/ufs_bsg.c @@ -232,6 +232,8 @@ static inline void ufs_bsg_node_release(struct device *dev) * @hba: per adapter object * * Called during initial loading of the driver, and before scsi_scan_host. + * + * Returns: 0 (success). */ int ufs_bsg_probe(struct ufs_hba *hba) { diff --git a/drivers/ufs/core/ufshcd-crypto.h b/drivers/ufs/core/ufshcd-crypto.h index 504cc841540b..be8596f20ba2 100644 --- a/drivers/ufs/core/ufshcd-crypto.h +++ b/drivers/ufs/core/ufshcd-crypto.h @@ -26,15 +26,15 @@ static inline void ufshcd_prepare_lrbp_crypto(struct request *rq, } static inline void -ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, u32 *dword_0, - u32 *dword_1, u32 *dword_3) +ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, + struct request_desc_header *h) { - if (lrbp->crypto_key_slot >= 0) { - *dword_0 |= UTP_REQ_DESC_CRYPTO_ENABLE_CMD; - *dword_0 |= lrbp->crypto_key_slot; - *dword_1 = lower_32_bits(lrbp->data_unit_num); - *dword_3 = upper_32_bits(lrbp->data_unit_num); - } + if (lrbp->crypto_key_slot < 0) + return; + h->enable_crypto = 1; + h->cci = lrbp->crypto_key_slot; + h->dunl = cpu_to_le32(lower_32_bits(lrbp->data_unit_num)); + h->dunu = cpu_to_le32(upper_32_bits(lrbp->data_unit_num)); } bool ufshcd_crypto_enable(struct ufs_hba *hba); @@ -51,8 +51,8 @@ static inline void ufshcd_prepare_lrbp_crypto(struct request *rq, struct ufshcd_lrb *lrbp) { } static inline void -ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, u32 *dword_0, - u32 *dword_1, u32 *dword_3) { } +ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp, + struct request_desc_header *h) { } static inline bool ufshcd_crypto_enable(struct ufs_hba *hba) { diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h index 0f3bd943b58b..f42d99ce5bf1 100644 --- a/drivers/ufs/core/ufshcd-priv.h +++ b/drivers/ufs/core/ufshcd-priv.h @@ -93,7 +93,7 @@ int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd); int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, struct utp_upiu_req *rsp_upiu, - int msgcode, + enum upiu_request_transaction msgcode, u8 *desc_buff, int *buff_len, enum query_opcode desc_op); @@ -294,7 +294,7 @@ extern const struct ufs_pm_lvl_states ufs_pm_lvl_states[]; * ufshcd_scsi_to_upiu_lun - maps scsi LUN to UPIU LUN * @scsi_lun: scsi LUN id * - * Returns UPIU LUN id + * Return: UPIU LUN id */ static inline u8 ufshcd_scsi_to_upiu_lun(unsigned int scsi_lun) { diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 129446775796..e4318171381b 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -34,7 +34,6 @@ #include "ufs-fault-injection.h" #include "ufs_bsg.h" #include "ufshcd-crypto.h" -#include "ufshpb.h" #include <asm/unaligned.h> #define CREATE_TRACE_POINTS @@ -238,8 +237,7 @@ static const struct ufs_dev_quirk ufs_fixups[] = { /* UFS cards deviations table */ { .wmanufacturerid = UFS_VENDOR_MICRON, .model = UFS_ANY_MODEL, - .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | - UFS_DEVICE_QUIRK_SWAP_L2P_ENTRY_FOR_HPB_READ }, + .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, { .wmanufacturerid = UFS_VENDOR_SAMSUNG, .model = UFS_ANY_MODEL, .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | @@ -703,8 +701,7 @@ EXPORT_SYMBOL_GPL(ufshcd_delay_us); * @interval_us: polling interval in microseconds * @timeout_ms: timeout in milliseconds * - * Return: - * -ETIMEDOUT on error, zero on success. + * Return: -ETIMEDOUT on error, zero on success. */ static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, u32 val, unsigned long interval_us, @@ -732,7 +729,7 @@ static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, * ufshcd_get_intr_mask - Get the interrupt bit mask * @hba: Pointer to adapter instance * - * Returns interrupt bit mask per version + * Return: interrupt bit mask per version */ static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) { @@ -748,7 +745,7 @@ static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) * ufshcd_get_ufs_version - Get the UFS version supported by the HBA * @hba: Pointer to adapter instance * - * Returns UFSHCI version supported by the controller + * Return: UFSHCI version supported by the controller */ static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) { @@ -775,7 +772,7 @@ static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) * the host controller * @hba: pointer to adapter instance * - * Returns true if device present, false if no device detected + * Return: true if device present, false if no device detected */ static inline bool ufshcd_is_device_present(struct ufs_hba *hba) { @@ -788,7 +785,8 @@ static inline bool ufshcd_is_device_present(struct ufs_hba *hba) * @cqe: pointer to the completion queue entry * * This function is used to get the OCS field from UTRD - * Returns the OCS field in the UTRD + * + * Return: the OCS field in the UTRD. */ static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp, struct cq_entry *cqe) @@ -796,7 +794,7 @@ static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp, if (cqe) return le32_to_cpu(cqe->status) & MASK_OCS; - return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS; + return lrbp->utr_descriptor_ptr->header.ocs & MASK_OCS; } /** @@ -841,7 +839,7 @@ static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos) * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY * @reg: Register value of host controller status * - * Returns integer, 0 on Success and positive value if failed + * Return: 0 on success; a positive value if failed. */ static inline int ufshcd_get_lists_status(u32 reg) { @@ -853,7 +851,8 @@ static inline int ufshcd_get_lists_status(u32 reg) * @hba: Pointer to adapter instance * * This function gets the result of UIC command completion - * Returns 0 on success, non zero value on error + * + * Return: 0 on success; non-zero value on error. */ static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) { @@ -866,7 +865,8 @@ static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) * @hba: Pointer to adapter instance * * This function gets UIC command argument3 - * Returns 0 on success, non zero value on error + * + * Return: 0 on success; non-zero value on error. */ static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) { @@ -876,38 +876,13 @@ static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) /** * ufshcd_get_req_rsp - returns the TR response transaction type * @ucd_rsp_ptr: pointer to response UPIU - */ -static inline int -ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) -{ - return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24; -} - -/** - * ufshcd_get_rsp_upiu_result - Get the result from response UPIU - * @ucd_rsp_ptr: pointer to response UPIU - * - * This function gets the response status and scsi_status from response UPIU - * Returns the response result code. - */ -static inline int -ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr) -{ - return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT; -} - -/* - * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length - * from response UPIU - * @ucd_rsp_ptr: pointer to response UPIU * - * Return the data segment length. + * Return: UPIU type. */ -static inline unsigned int -ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr) +static inline enum upiu_response_transaction +ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) { - return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & - MASK_RSP_UPIU_DATA_SEG_LEN; + return ucd_rsp_ptr->header.transaction_code; } /** @@ -917,12 +892,11 @@ ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr) * The function checks if the device raised an exception event indicated in * the Device Information field of response UPIU. * - * Returns true if exception is raised, false otherwise. + * Return: true if exception is raised, false otherwise. */ static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr) { - return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & - MASK_RSP_EXCEPTION_EVENT; + return ucd_rsp_ptr->header.device_information & 1; } /** @@ -993,12 +967,13 @@ static inline void ufshcd_hba_start(struct ufs_hba *hba) * ufshcd_is_hba_active - Get controller state * @hba: per adapter instance * - * Returns true if and only if the controller is active. + * Return: true if and only if the controller is active. */ -static inline bool ufshcd_is_hba_active(struct ufs_hba *hba) +bool ufshcd_is_hba_active(struct ufs_hba *hba) { return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE; } +EXPORT_SYMBOL_GPL(ufshcd_is_hba_active); u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba) { @@ -1029,8 +1004,7 @@ static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba) * @hba: per adapter instance * @scale_up: If True, set max possible frequency othewise set low frequency * - * Returns 0 if successful - * Returns < 0 for any other errors + * Return: 0 if successful; < 0 upon failure. */ static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) { @@ -1092,8 +1066,7 @@ out: * @hba: per adapter instance * @scale_up: True if scaling up and false if scaling down * - * Returns 0 if successful - * Returns < 0 for any other errors + * Return: 0 if successful; < 0 upon failure. */ static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up) { @@ -1124,7 +1097,7 @@ out: * @hba: per adapter instance * @scale_up: True if scaling up and false if scaling down * - * Returns true if scaling is required, false otherwise. + * Return: true if scaling is required, false otherwise. */ static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, bool scale_up) @@ -1241,9 +1214,8 @@ out: * @hba: per adapter instance * @scale_up: True for scaling up gear and false for scaling down * - * Returns 0 for success, - * Returns -EBUSY if scaling can't happen at this time - * Returns non-zero for any other errors + * Return: 0 for success; -EBUSY if scaling can't happen at this time; + * non-zero for any other errors. */ static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up) { @@ -1333,9 +1305,8 @@ static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err, bool sc * @hba: per adapter instance * @scale_up: True for scaling up and false for scalin down * - * Returns 0 for success, - * Returns -EBUSY if scaling can't happen at this time - * Returns non-zero for any other errors + * Return: 0 for success; -EBUSY if scaling can't happen at this time; non-zero + * for any other errors. */ static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up) { @@ -2225,10 +2196,11 @@ void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag, static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) { u8 *const sense_buffer = lrbp->cmd->sense_buffer; + u16 resp_len; int len; - if (sense_buffer && - ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) { + resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header.data_segment_length); + if (sense_buffer && resp_len) { int len_to_copy; len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len); @@ -2244,6 +2216,8 @@ static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) * descriptor * @hba: per adapter instance * @lrbp: pointer to local reference block + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) @@ -2261,8 +2235,8 @@ int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) u16 buf_len; /* data segment length */ - resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & - MASK_QUERY_DATA_SEG_LEN; + resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header + .data_segment_length); buf_len = be16_to_cpu( hba->dev_cmd.query.request.upiu_req.length); if (likely(buf_len >= resp_len)) { @@ -2320,7 +2294,8 @@ static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) * ufshcd_ready_for_uic_cmd - Check if controller is ready * to accept UIC commands * @hba: per adapter instance - * Return true on success, else false + * + * Return: true on success, else false. */ static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) { @@ -2332,7 +2307,8 @@ static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) * @hba: Pointer to adapter instance * * This function gets the UPMCRS field of HCS register - * Returns value of UPMCRS field + * + * Return: value of UPMCRS field. */ static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba) { @@ -2370,7 +2346,7 @@ ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) * @hba: per adapter instance * @uic_cmd: UIC command * - * Returns 0 only if success. + * Return: 0 only if success. */ static int ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) @@ -2409,7 +2385,7 @@ ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) * @uic_cmd: UIC command * @completion: initialize the completion only if this is set to true * - * Returns 0 only if success. + * Return: 0 only if success. */ static int __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd, @@ -2438,7 +2414,7 @@ __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd, * @hba: per adapter instance * @uic_cmd: UIC command * - * Returns 0 only if success. + * Return: 0 only if success. */ int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) { @@ -2515,7 +2491,7 @@ static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int * @hba: per adapter instance * @lrbp: pointer to local reference block * - * Returns 0 in case of success, non-zero value in case of failure + * Return: 0 in case of success, non-zero value in case of failure. */ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { @@ -2584,10 +2560,10 @@ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, u8 *upiu_flags, enum dma_data_direction cmd_dir, int ehs_length) { struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr; - u32 data_direction; - u32 dword_0; - u32 dword_1 = 0; - u32 dword_3 = 0; + struct request_desc_header *h = &req_desc->header; + enum utp_data_direction data_direction; + + *h = (typeof(*h)){ }; if (cmd_dir == DMA_FROM_DEVICE) { data_direction = UTP_DEVICE_TO_HOST; @@ -2600,25 +2576,22 @@ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, u8 *upiu_flags, *upiu_flags = UPIU_CMD_FLAGS_NONE; } - dword_0 = data_direction | (lrbp->command_type << UPIU_COMMAND_TYPE_OFFSET) | - ehs_length << 8; + h->command_type = lrbp->command_type; + h->data_direction = data_direction; + h->ehs_length = ehs_length; + if (lrbp->intr_cmd) - dword_0 |= UTP_REQ_DESC_INT_CMD; + h->interrupt = 1; /* Prepare crypto related dwords */ - ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3); + ufshcd_prepare_req_desc_hdr_crypto(lrbp, h); - /* Transfer request descriptor header fields */ - req_desc->header.dword_0 = cpu_to_le32(dword_0); - req_desc->header.dword_1 = cpu_to_le32(dword_1); /* * assigning invalid value for command status. Controller * updates OCS on command completion, with the command * status */ - req_desc->header.dword_2 = - cpu_to_le32(OCS_INVALID_COMMAND_STATUS); - req_desc->header.dword_3 = cpu_to_le32(dword_3); + h->ocs = OCS_INVALID_COMMAND_STATUS; req_desc->prd_table_length = 0; } @@ -2636,15 +2609,13 @@ void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags) struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; unsigned short cdb_len; - /* command descriptor fields */ - ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( - UPIU_TRANSACTION_COMMAND, upiu_flags, - lrbp->lun, lrbp->task_tag); - ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( - UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0); - - /* Total EHS length and Data segment length will be zero */ - ucd_req_ptr->header.dword_2 = 0; + ucd_req_ptr->header = (struct utp_upiu_header){ + .transaction_code = UPIU_TRANSACTION_COMMAND, + .flags = upiu_flags, + .lun = lrbp->lun, + .task_tag = lrbp->task_tag, + .command_set_type = UPIU_COMMAND_SET_TYPE_SCSI, + }; ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length); @@ -2669,18 +2640,19 @@ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, u16 len = be16_to_cpu(query->request.upiu_req.length); /* Query request header */ - ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( - UPIU_TRANSACTION_QUERY_REQ, upiu_flags, - lrbp->lun, lrbp->task_tag); - ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( - 0, query->request.query_func, 0, 0); - - /* Data segment length only need for WRITE_DESC */ - if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) - ucd_req_ptr->header.dword_2 = - UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len); - else - ucd_req_ptr->header.dword_2 = 0; + ucd_req_ptr->header = (struct utp_upiu_header){ + .transaction_code = UPIU_TRANSACTION_QUERY_REQ, + .flags = upiu_flags, + .lun = lrbp->lun, + .task_tag = lrbp->task_tag, + .query_function = query->request.query_func, + /* Data segment length only need for WRITE_DESC */ + .data_segment_length = + query->request.upiu_req.opcode == + UPIU_QUERY_OPCODE_WRITE_DESC ? + cpu_to_be16(len) : + 0, + }; /* Copy the Query Request buffer as is */ memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, @@ -2699,13 +2671,10 @@ static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); - /* command descriptor fields */ - ucd_req_ptr->header.dword_0 = - UPIU_HEADER_DWORD( - UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag); - /* clear rest of the fields of basic header */ - ucd_req_ptr->header.dword_1 = 0; - ucd_req_ptr->header.dword_2 = 0; + ucd_req_ptr->header = (struct utp_upiu_header){ + .transaction_code = UPIU_TRANSACTION_NOP_OUT, + .task_tag = lrbp->task_tag, + }; memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); } @@ -2715,6 +2684,8 @@ static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) * for Device Management Purposes * @hba: per adapter instance * @lrbp: pointer to local reference block + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) @@ -2743,6 +2714,8 @@ static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, * for SCSI Purposes * @hba: per adapter instance * @lrbp: pointer to local reference block + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { @@ -2768,7 +2741,7 @@ static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID * @upiu_wlun_id: UPIU W-LUN id * - * Returns SCSI W-LUN id + * Return: SCSI W-LUN id. */ static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id) { @@ -2839,7 +2812,7 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) * @host: SCSI host pointer * @cmd: command from SCSI Midlayer * - * Returns 0 for success, non-zero in case of failure + * Return: 0 for success, non-zero in case of failure. */ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) { @@ -2908,8 +2881,6 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) lrbp->req_abort_skip = false; - ufshpb_prep(hba, lrbp); - ufshcd_comp_scsi_upiu(hba, lrbp); err = ufshcd_map_sg(hba, lrbp); @@ -2952,7 +2923,7 @@ static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, * Check with the block layer if the command is inflight * @cmd: command to check. * - * Returns true if command is inflight; false if not. + * Return: true if command is inflight; false if not. */ bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd) { @@ -3007,26 +2978,17 @@ static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag) mask, ~mask, 1000, 1000); } -static int -ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) -{ - struct ufs_query_res *query_res = &hba->dev_cmd.query.response; - - /* Get the UPIU response */ - query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >> - UPIU_RSP_CODE_OFFSET; - return query_res->response; -} - /** * ufshcd_dev_cmd_completion() - handles device management command responses * @hba: per adapter instance * @lrbp: pointer to local reference block + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { - int resp; + enum upiu_response_transaction resp; int err = 0; hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); @@ -3040,11 +3002,13 @@ ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) __func__, resp); } break; - case UPIU_TRANSACTION_QUERY_RSP: - err = ufshcd_check_query_response(hba, lrbp); - if (!err) + case UPIU_TRANSACTION_QUERY_RSP: { + u8 response = lrbp->ucd_rsp_ptr->header.response; + + if (response == 0) err = ufshcd_copy_query_response(hba, lrbp); break; + } case UPIU_TRANSACTION_REJECT_UPIU: /* TODO: handle Reject UPIU Response */ err = -EPERM; @@ -3159,6 +3123,8 @@ retry: * @cmd_type: specifies the type (NOP, Query...) * @timeout: timeout in milliseconds * + * Return: 0 upon success; < 0 upon failure. + * * NOTE: Since there is only one available tag for device management commands, * it is expected you hold the hba->dev_cmd.lock mutex. */ @@ -3250,7 +3216,7 @@ static int ufshcd_query_flag_retry(struct ufs_hba *hba, * @index: flag index to access * @flag_res: the flag value after the query request completes * - * Returns 0 for success, non-zero in case of failure + * Return: 0 for success, non-zero in case of failure. */ int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) @@ -3319,7 +3285,7 @@ out_unlock: * @selector: selector field * @attr_val: the attribute value after the query request completes * - * Returns 0 for success, non-zero in case of failure + * Return: 0 for success, non-zero in case of failure. */ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) @@ -3384,7 +3350,7 @@ out_unlock: * @attr_val: the attribute value after the query request * completes * - * Returns 0 for success, non-zero in case of failure + * Return: 0 for success, non-zero in case of failure. */ int ufshcd_query_attr_retry(struct ufs_hba *hba, enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, @@ -3482,9 +3448,10 @@ out_unlock: * @desc_buf: the buffer that contains the descriptor * @buf_len: length parameter passed to the device * - * Returns 0 for success, non-zero in case of failure. * The buf_len parameter will contain, on return, the length parameter * received on the response. + * + * Return: 0 for success, non-zero in case of failure. */ int ufshcd_query_descriptor_retry(struct ufs_hba *hba, enum query_opcode opcode, @@ -3514,7 +3481,7 @@ int ufshcd_query_descriptor_retry(struct ufs_hba *hba, * @param_read_buf: pointer to buffer where parameter would be read * @param_size: sizeof(param_read_buf) * - * Return 0 in case of success, non-zero otherwise + * Return: 0 in case of success, non-zero otherwise. */ int ufshcd_read_desc_param(struct ufs_hba *hba, enum desc_idn desc_id, @@ -3694,7 +3661,7 @@ out: * @param_read_buf: pointer to buffer where parameter would be read * @param_size: sizeof(param_read_buf) * - * Return 0 in case of success, non-zero otherwise + * Return: 0 in case of success, non-zero otherwise. */ static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba, int lun, @@ -3749,7 +3716,7 @@ static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba) * (UTMRDL) * 4. Allocate memory for local reference block(lrb). * - * Returns 0 for success, non-zero in case of failure + * Return: 0 for success, non-zero in case of failure. */ static int ufshcd_memory_alloc(struct ufs_hba *hba) { @@ -3896,7 +3863,7 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba) * Once the Unipro links are up, the device connected to the controller * is detected. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int ufshcd_dme_link_startup(struct ufs_hba *hba) { @@ -3918,7 +3885,7 @@ static int ufshcd_dme_link_startup(struct ufs_hba *hba) * DME_RESET command is issued in order to reset UniPro stack. * This function now deals with cold reset. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int ufshcd_dme_reset(struct ufs_hba *hba) { @@ -3957,7 +3924,7 @@ EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt); * * DME_ENABLE command is issued in order to enable UniPro stack. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int ufshcd_dme_enable(struct ufs_hba *hba) { @@ -4013,7 +3980,7 @@ static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) * @mib_val: setting value as uic command argument3 * @peer: indicate whether peer or local * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, u8 attr_set, u32 mib_val, u8 peer) @@ -4057,7 +4024,7 @@ EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr); * @mib_val: the value of the attribute as returned by the UIC command * @peer: indicate whether peer or local * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, u32 *mib_val, u8 peer) @@ -4138,7 +4105,7 @@ EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr); * addition to normal UIC command completion Status (UCCS). This function only * returns after the relevant status bits indicate the completion. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) { @@ -4228,7 +4195,7 @@ out_unlock: * @hba: per adapter instance * @mode: powr mode value * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) { @@ -4380,8 +4347,8 @@ static void ufshcd_init_pwr_info(struct ufs_hba *hba) { hba->pwr_info.gear_rx = UFS_PWM_G1; hba->pwr_info.gear_tx = UFS_PWM_G1; - hba->pwr_info.lane_rx = 1; - hba->pwr_info.lane_tx = 1; + hba->pwr_info.lane_rx = UFS_LANE_1; + hba->pwr_info.lane_tx = UFS_LANE_1; hba->pwr_info.pwr_rx = SLOWAUTO_MODE; hba->pwr_info.pwr_tx = SLOWAUTO_MODE; hba->pwr_info.hs_rate = 0; @@ -4390,6 +4357,8 @@ static void ufshcd_init_pwr_info(struct ufs_hba *hba) /** * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device * @hba: per-adapter instance + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) { @@ -4547,6 +4516,8 @@ static int ufshcd_change_power_mode(struct ufs_hba *hba, * ufshcd_config_pwr_mode - configure a new power mode * @hba: per-adapter instance * @desired_pwr_mode: desired power configuration + * + * Return: 0 upon success; < 0 upon failure. */ int ufshcd_config_pwr_mode(struct ufs_hba *hba, struct ufs_pa_layer_attr *desired_pwr_mode) @@ -4571,6 +4542,8 @@ EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode); * @hba: per-adapter instance * * Set fDeviceInit flag and poll until device toggles it. + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_complete_dev_init(struct ufs_hba *hba) { @@ -4621,7 +4594,7 @@ out: * 3. Program UTRL and UTMRL base address * 4. Configure run-stop-registers * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ int ufshcd_make_hba_operational(struct ufs_hba *hba) { @@ -4702,7 +4675,7 @@ EXPORT_SYMBOL_GPL(ufshcd_hba_stop); * sequence kicks off. When controller is ready it will set * the Host Controller Enable bit to 1. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int ufshcd_hba_execute_hce(struct ufs_hba *hba) { @@ -4847,7 +4820,7 @@ EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist); * ufshcd_link_startup - Initialize unipro link startup * @hba: per adapter instance * - * Returns 0 for success, non-zero in case of failure + * Return: 0 for success, non-zero in case of failure. */ static int ufshcd_link_startup(struct ufs_hba *hba) { @@ -4942,6 +4915,8 @@ out: * If the UTP layer at the device side is not initialized, it may * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations. + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_verify_dev_init(struct ufs_hba *hba) { @@ -5066,7 +5041,7 @@ set_qdepth: * ufshcd_slave_alloc - handle initial SCSI device configurations * @sdev: pointer to SCSI device * - * Returns success + * Return: success. */ static int ufshcd_slave_alloc(struct scsi_device *sdev) { @@ -5102,43 +5077,25 @@ static int ufshcd_slave_alloc(struct scsi_device *sdev) * @depth: required depth to set * * Change queue depth and make sure the max. limits are not crossed. + * + * Return: new queue depth. */ static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth) { return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue)); } -static void ufshcd_hpb_destroy(struct ufs_hba *hba, struct scsi_device *sdev) -{ - /* skip well-known LU */ - if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) || - !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba)) - return; - - ufshpb_destroy_lu(hba, sdev); -} - -static void ufshcd_hpb_configure(struct ufs_hba *hba, struct scsi_device *sdev) -{ - /* skip well-known LU */ - if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) || - !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba)) - return; - - ufshpb_init_hpb_lu(hba, sdev); -} - /** * ufshcd_slave_configure - adjust SCSI device configurations * @sdev: pointer to SCSI device + * + * Return: 0 (success). */ static int ufshcd_slave_configure(struct scsi_device *sdev) { struct ufs_hba *hba = shost_priv(sdev->host); struct request_queue *q = sdev->request_queue; - ufshcd_hpb_configure(hba, sdev); - blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1); if (hba->quirks & UFSHCD_QUIRK_4KB_DMA_ALIGNMENT) blk_queue_update_dma_alignment(q, SZ_4K - 1); @@ -5173,8 +5130,6 @@ static void ufshcd_slave_destroy(struct scsi_device *sdev) hba = shost_priv(sdev->host); - ufshcd_hpb_destroy(hba, sdev); - /* Drop the reference as it won't be needed anymore */ if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) { spin_lock_irqsave(hba->host->host_lock, flags); @@ -5208,7 +5163,7 @@ static void ufshcd_slave_destroy(struct scsi_device *sdev) * @lrbp: pointer to local reference block of completed command * @scsi_status: SCSI command status * - * Returns value base on SCSI command status + * Return: value base on SCSI command status. */ static inline int ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) @@ -5242,7 +5197,7 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) * @lrbp: pointer to local reference block of completed command * @cqe: pointer to the completion queue entry * - * Returns result of the command to notify SCSI midlayer + * Return: result of the command to notify SCSI midlayer. */ static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, @@ -5251,36 +5206,37 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int result = 0; int scsi_status; enum utp_ocs ocs; + u8 upiu_flags; + u32 resid; - scsi_set_resid(lrbp->cmd, - be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count)); + upiu_flags = lrbp->ucd_rsp_ptr->header.flags; + resid = be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count); + /* + * Test !overflow instead of underflow to support UFS devices that do + * not set either flag. + */ + if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW)) + scsi_set_resid(lrbp->cmd, resid); /* overall command status of utrd */ ocs = ufshcd_get_tr_ocs(lrbp, cqe); if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) { - if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) & - MASK_RSP_UPIU_RESULT) + if (lrbp->ucd_rsp_ptr->header.response || + lrbp->ucd_rsp_ptr->header.status) ocs = OCS_SUCCESS; } switch (ocs) { case OCS_SUCCESS: - result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); - switch (result) { + switch (ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr)) { case UPIU_TRANSACTION_RESPONSE: /* - * get the response UPIU result to extract - * the SCSI command status - */ - result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr); - - /* * get the result based on SCSI status response * to notify the SCSI midlayer of the command status */ - scsi_status = result & MASK_SCSI_STATUS; + scsi_status = lrbp->ucd_rsp_ptr->header.status; result = ufshcd_scsi_cmd_status(lrbp, scsi_status); /* @@ -5300,9 +5256,6 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, ufshcd_is_exception_event(lrbp->ucd_rsp_ptr)) /* Flushed in suspend */ schedule_work(&hba->eeh_work); - - if (scsi_status == SAM_STAT_GOOD) - ufshpb_rsp_upiu(hba, lrbp); break; case UPIU_TRANSACTION_REJECT_UPIU: /* TODO: handle Reject UPIU Response */ @@ -5372,7 +5325,7 @@ static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba, * @hba: per adapter instance * @intr_status: interrupt status generated by the controller * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -5448,8 +5401,7 @@ void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, if (hba->dev_cmd.complete) { if (cqe) { ocs = le32_to_cpu(cqe->status) & MASK_OCS; - lrbp->utr_descriptor_ptr->header.dword_2 = - cpu_to_le32(ocs); + lrbp->utr_descriptor_ptr->header.ocs = ocs; } complete(hba->dev_cmd.complete); ufshcd_clk_scaling_update_busy(hba); @@ -5492,7 +5444,7 @@ static void ufshcd_clear_polled(struct ufs_hba *hba, } /* - * Returns > 0 if one or more commands have been completed or 0 if no + * Return: > 0 if one or more commands have been completed or 0 if no * requests have been completed. */ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) @@ -5582,7 +5534,7 @@ static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, * ufshcd_transfer_req_compl - handle SCSI and query command completion * @hba: per adapter instance * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -5659,7 +5611,7 @@ int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, * Disables exception event in the device so that the EVENT_ALERT * bit is not set. * - * Returns zero on success, non-zero error value on failure. + * Return: zero on success, non-zero error value on failure. */ static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) { @@ -5674,7 +5626,7 @@ static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) * Enable corresponding exception event in the device to allow * device to alert host in critical scenarios. * - * Returns zero on success, non-zero error value on failure. + * Return: zero on success, non-zero error value on failure. */ static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) { @@ -5690,7 +5642,7 @@ static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) * as the device is allowed to manage its own way of handling background * operations. * - * Returns zero on success, non-zero on failure. + * Return: zero on success, non-zero on failure. */ static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) { @@ -5729,7 +5681,7 @@ out: * host is idle so that BKOPS are managed effectively without any negative * impacts. * - * Returns zero on success, non-zero on failure. + * Return: zero on success, non-zero on failure. */ static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) { @@ -5805,7 +5757,7 @@ static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status) * bkops_status is greater than or equal to "status" argument passed to * this function, disable otherwise. * - * Returns 0 for success, non-zero in case of failure. + * Return: 0 for success, non-zero in case of failure. * * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag * to know whether auto bkops is enabled or disabled after this function @@ -5846,6 +5798,8 @@ out: * * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled * and negative error value for any other failure. + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_urgent_bkops(struct ufs_hba *hba) { @@ -6157,7 +6111,7 @@ static void ufshcd_complete_requests(struct ufs_hba *hba, bool force_compl) * to recover from the DL NAC errors or not. * @hba: per-adapter instance * - * Returns true if error handling is required, false otherwise + * Return: true if error handling is required, false otherwise. */ static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba) { @@ -6384,54 +6338,48 @@ static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba) return false; } +static bool ufshcd_abort_one(struct request *rq, void *priv) +{ + int *ret = priv; + u32 tag = rq->tag; + struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); + struct scsi_device *sdev = cmd->device; + struct Scsi_Host *shost = sdev->host; + struct ufs_hba *hba = shost_priv(shost); + + *ret = ufshcd_try_to_abort_task(hba, tag); + dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, + hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1, + *ret ? "failed" : "succeeded"); + return *ret == 0; +} + +/** + * ufshcd_abort_all - Abort all pending commands. + * @hba: Host bus adapter pointer. + * + * Return: true if and only if the host controller needs to be reset. + */ static bool ufshcd_abort_all(struct ufs_hba *hba) { - bool needs_reset = false; - int tag, ret; + int tag, ret = 0; - if (is_mcq_enabled(hba)) { - struct ufshcd_lrb *lrbp; - int tag; + blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_abort_one, &ret); + if (ret) + goto out; - for (tag = 0; tag < hba->nutrs; tag++) { - lrbp = &hba->lrb[tag]; - if (!ufshcd_cmd_inflight(lrbp->cmd)) - continue; - ret = ufshcd_try_to_abort_task(hba, tag); - dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, - hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1, - ret ? "failed" : "succeeded"); - if (ret) { - needs_reset = true; - goto out; - } - } - } else { - /* Clear pending transfer requests */ - for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) { - ret = ufshcd_try_to_abort_task(hba, tag); - dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, - hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1, - ret ? "failed" : "succeeded"); - if (ret) { - needs_reset = true; - goto out; - } - } - } /* Clear pending task management requests */ for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) { - if (ufshcd_clear_tm_cmd(hba, tag)) { - needs_reset = true; + ret = ufshcd_clear_tm_cmd(hba, tag); + if (ret) goto out; - } } out: /* Complete the requests that are cleared by s/w */ ufshcd_complete_requests(hba, false); - return needs_reset; + return ret != 0; } /** @@ -6618,7 +6566,7 @@ skip_err_handling: * ufshcd_update_uic_error - check and set fatal UIC error flags. * @hba: per-adapter instance * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -6711,7 +6659,7 @@ static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba) * @hba: per-adapter instance * @intr_status: interrupt status generated by the controller * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -6787,7 +6735,7 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) * ufshcd_tmc_handler - handle task management function completion * @hba: per adapter instance * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -6816,7 +6764,7 @@ static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba) * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events * @hba: per adapter instance * - * Returns IRQ_HANDLED if interrupt is handled + * Return: IRQ_HANDLED if interrupt is handled. */ static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba) { @@ -6851,7 +6799,7 @@ static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba) * @hba: per adapter instance * @intr_status: contains interrupts generated by the controller * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -6882,7 +6830,7 @@ static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) * @irq: irq number * @__hba: pointer to adapter instance * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -6978,7 +6926,7 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba, WARN_ONCE(task_tag < 0 || task_tag >= hba->nutmrs, "Invalid tag %d\n", task_tag); hba->tmf_rqs[req->tag] = req; - treq->upiu_req.req_header.dword_0 |= cpu_to_be32(task_tag); + treq->upiu_req.req_header.task_tag = task_tag; memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq)); ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function); @@ -7031,23 +6979,23 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba, * @tm_function: task management function opcode * @tm_response: task management service response return value * - * Returns non-zero value on error, zero on success. + * Return: non-zero value on error, zero on success. */ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, u8 tm_function, u8 *tm_response) { - struct utp_task_req_desc treq = { { 0 }, }; + struct utp_task_req_desc treq = { }; enum utp_ocs ocs_value; int err; /* Configure task request descriptor */ - treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD); - treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS); + treq.header.interrupt = 1; + treq.header.ocs = OCS_INVALID_COMMAND_STATUS; /* Configure task request UPIU */ - treq.upiu_req.req_header.dword_0 = cpu_to_be32(lun_id << 8) | - cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24); - treq.upiu_req.req_header.dword_1 = cpu_to_be32(tm_function << 16); + treq.upiu_req.req_header.transaction_code = UPIU_TRANSACTION_TASK_REQ; + treq.upiu_req.req_header.lun = lun_id; + treq.upiu_req.req_header.tm_function = tm_function; /* * The host shall provide the same value for LUN field in the basic @@ -7060,7 +7008,7 @@ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, if (err == -ETIMEDOUT) return err; - ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS; + ocs_value = treq.header.ocs & MASK_OCS; if (ocs_value != OCS_SUCCESS) dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__, ocs_value); @@ -7086,6 +7034,8 @@ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, * * Since there is only one available tag for device management commands, * the caller is expected to hold the hba->dev_cmd.lock mutex. + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, @@ -7119,7 +7069,7 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; /* update the task tag in the request upiu */ - req_upiu->header.dword_0 |= cpu_to_be32(tag); + req_upiu->header.task_tag = tag; ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE, 0); @@ -7152,8 +7102,8 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) { u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu); - u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & - MASK_QUERY_DATA_SEG_LEN; + u16 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header + .data_segment_length); if (*buff_len >= resp_len) { memcpy(desc_buff, descp, resp_len); @@ -7187,19 +7137,21 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, * Management requests. * It is up to the caller to fill the upiu conent properly, as it will * be copied without any further input validations. + * + * Return: 0 upon success; < 0 upon failure. */ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, struct utp_upiu_req *rsp_upiu, - int msgcode, + enum upiu_request_transaction msgcode, u8 *desc_buff, int *buff_len, enum query_opcode desc_op) { int err; enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY; - struct utp_task_req_desc treq = { { 0 }, }; + struct utp_task_req_desc treq = { }; enum utp_ocs ocs_value; - u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC; + u8 tm_f = req_upiu->header.tm_function; switch (msgcode) { case UPIU_TRANSACTION_NOP_OUT: @@ -7216,8 +7168,8 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, break; case UPIU_TRANSACTION_TASK_REQ: - treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD); - treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS); + treq.header.interrupt = 1; + treq.header.ocs = OCS_INVALID_COMMAND_STATUS; memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu)); @@ -7225,7 +7177,7 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, if (err == -ETIMEDOUT) break; - ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS; + ocs_value = treq.header.ocs & MASK_OCS; if (ocs_value != OCS_SUCCESS) { dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__, ocs_value); @@ -7255,7 +7207,7 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, * @sg_list: Pointer to SG list when DATA IN/OUT UPIU is required in ARPMB operation * @dir: DMA direction * - * Returns zero on success, non-zero on failure + * Return: zero on success, non-zero on failure. */ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, struct utp_upiu_req *rsp_upiu, struct ufs_ehs *req_ehs, @@ -7291,7 +7243,9 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, dir, 2); /* update the task tag and LUN in the request upiu */ - req_upiu->header.dword_0 |= cpu_to_be32(upiu_flags << 16 | UFS_UPIU_RPMB_WLUN << 8 | tag); + req_upiu->header.flags = upiu_flags; + req_upiu->header.lun = UFS_UPIU_RPMB_WLUN; + req_upiu->header.task_tag = tag; /* copy the UPIU(contains CDB) request as it is */ memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); @@ -7313,9 +7267,10 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r /* Just copy the upiu response as it is */ memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); /* Get the response UPIU result */ - result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr); + result = (lrbp->ucd_rsp_ptr->header.response << 8) | + lrbp->ucd_rsp_ptr->header.status; - ehs_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) >> 24; + ehs_len = lrbp->ucd_rsp_ptr->header.ehs_length; /* * Since the bLength in EHS indicates the total size of the EHS Header and EHS Data * in 32 Byte units, the value of the bLength Request/Response for Advanced RPMB @@ -7341,7 +7296,7 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r * ufshcd_eh_device_reset_handler() - Reset a single logical unit. * @cmd: SCSI command pointer * - * Returns SUCCESS/FAILED + * Return: SUCCESS or FAILED. */ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) { @@ -7436,7 +7391,7 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is * really issued and then try to abort it. * - * Returns zero on success, non-zero on failure + * Return: zero on success, non-zero on failure. */ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) { @@ -7524,7 +7479,7 @@ out: * ufshcd_abort - scsi host template eh_abort_handler callback * @cmd: SCSI command pointer * - * Returns SUCCESS/FAILED + * Return: SUCCESS or FAILED. */ static int ufshcd_abort(struct scsi_cmnd *cmd) { @@ -7649,7 +7604,7 @@ release: * local and remote (device) Uni-Pro stack and the attributes * are reset to default state. * - * Returns zero on success, non-zero on failure + * Return: zero on success, non-zero on failure. */ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) { @@ -7659,7 +7614,6 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) * Stop the host controller and complete the requests * cleared by h/w */ - ufshpb_toggle_state(hba, HPB_PRESENT, HPB_RESET); ufshcd_hba_stop(hba); hba->silence_err_logs = true; ufshcd_complete_requests(hba, true); @@ -7687,7 +7641,7 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) * Reset and recover device, host and re-establish link. This * is helpful to recover the communication in fatal error conditions. * - * Returns zero on success, non-zero on failure + * Return: zero on success, non-zero on failure. */ static int ufshcd_reset_and_restore(struct ufs_hba *hba) { @@ -7745,7 +7699,7 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba) * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer * @cmd: SCSI command pointer * - * Returns SUCCESS/FAILED + * Return: SUCCESS or FAILED. */ static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) { @@ -7777,7 +7731,7 @@ static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) * @start_scan: row at the desc table to start scan from * @buff: power descriptor buffer * - * Returns calculated max ICC level for specific regulator + * Return: calculated max ICC level for specific regulator. */ static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, const char *buff) @@ -7823,7 +7777,7 @@ static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, * @hba: per-adapter instance * @desc_buf: power descriptor buffer to extract ICC levels from. * - * Returns calculated ICC level + * Return: calculated ICC level. */ static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba, const u8 *desc_buf) @@ -7932,7 +7886,7 @@ static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev) * This function adds scsi device instances for each of all well known LUs * (except "REPORT LUNS" LU). * - * Returns zero on success (all required W-LUs are added successfully), + * Return: zero on success (all required W-LUs are added successfully), * non-zero error value on failure (if failed to add any of the required W-LU). */ static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) @@ -8122,7 +8076,6 @@ static int ufs_get_device_desc(struct ufs_hba *hba) { int err; u8 model_index; - u8 b_ufs_feature_sup; u8 *desc_buf; struct ufs_dev_info *dev_info = &hba->dev_info; @@ -8151,26 +8104,9 @@ static int ufs_get_device_desc(struct ufs_hba *hba) dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 | desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1]; dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH]; - b_ufs_feature_sup = desc_buf[DEVICE_DESC_PARAM_UFS_FEAT]; model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME]; - if (dev_info->wspecversion >= UFS_DEV_HPB_SUPPORT_VERSION && - (b_ufs_feature_sup & UFS_DEV_HPB_SUPPORT)) { - bool hpb_en = false; - - ufshpb_get_dev_info(hba, desc_buf); - - if (!ufshpb_is_legacy(hba)) - err = ufshcd_query_flag_retry(hba, - UPIU_QUERY_OPCODE_READ_FLAG, - QUERY_FLAG_IDN_HPB_EN, 0, - &hpb_en); - - if (ufshpb_is_legacy(hba) || (!err && hpb_en)) - dev_info->hpb_enabled = true; - } - err = ufshcd_read_string_desc(hba, model_index, &dev_info->model, SD_ASCII_STD); if (err < 0) { @@ -8219,7 +8155,7 @@ static void ufs_put_device_desc(struct ufs_hba *hba) * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce * the hibern8 exit latency. * - * Returns zero on success, non-zero error value on failure. + * Return: zero on success, non-zero error value on failure. */ static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba) { @@ -8254,7 +8190,7 @@ out: * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY. * This optimal value can help reduce the hibern8 exit latency. * - * Returns zero on success, non-zero error value on failure. + * Return: zero on success, non-zero error value on failure. */ static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba) { @@ -8296,7 +8232,7 @@ out: * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk * for such devices. * - * Returns zero on success, non-zero error value on failure. + * Return: zero on success, non-zero error value on failure. */ static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba) { @@ -8405,10 +8341,6 @@ static int ufshcd_device_geo_params_init(struct ufs_hba *hba) else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0) hba->dev_info.max_lu_supported = 8; - if (desc_buf[QUERY_DESC_LENGTH_OFFSET] >= - GEOMETRY_DESC_PARAM_HPB_MAX_ACTIVE_REGS) - ufshpb_get_geo_info(hba, desc_buf); - out: kfree(desc_buf); return err; @@ -8558,6 +8490,8 @@ static void ufshcd_set_timestamp_attr(struct ufs_hba *hba) /** * ufshcd_add_lus - probe and add UFS logical units * @hba: per-adapter instance + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_add_lus(struct ufs_hba *hba) { @@ -8584,7 +8518,6 @@ static int ufshcd_add_lus(struct ufs_hba *hba) } ufs_bsg_probe(hba); - ufshpb_init(hba); scsi_scan_host(hba->host); pm_runtime_put_sync(hba->dev); @@ -8770,6 +8703,8 @@ static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params) * @init_dev_params: whether or not to call ufshcd_device_params_init(). * * Execute link-startup and verify device initialization + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) { @@ -8818,7 +8753,6 @@ static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) /* Enable Auto-Hibernate if configured */ ufshcd_auto_hibern8_enable(hba); - ufshpb_toggle_state(hba, HPB_RESET, HPB_PRESENT); out: spin_lock_irqsave(hba->host->host_lock, flags); if (ret) @@ -8888,10 +8822,6 @@ static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd) static const struct attribute_group *ufshcd_driver_groups[] = { &ufs_sysfs_unit_descriptor_group, &ufs_sysfs_lun_attributes_group, -#ifdef CONFIG_SCSI_UFS_HPB - &ufs_sysfs_hpb_stat_group, - &ufs_sysfs_hpb_param_group, -#endif NULL, }; @@ -9235,8 +9165,9 @@ static int ufshcd_variant_hba_init(struct ufs_hba *hba) err = ufshcd_vops_init(hba); if (err) - dev_err(hba->dev, "%s: variant %s init failed err %d\n", - __func__, ufshcd_get_var_name(hba), err); + dev_err_probe(hba->dev, err, + "%s: variant %s init failed with err %d\n", + __func__, ufshcd_get_var_name(hba), err); out: return err; } @@ -9345,8 +9276,8 @@ static int ufshcd_execute_start_stop(struct scsi_device *sdev, * @hba: per adapter instance * @pwr_mode: device power mode to set * - * Returns 0 if requested power mode is set successfully - * Returns < 0 if failed to set the requested power mode + * Return: 0 if requested power mode is set successfully; + * < 0 if failed to set the requested power mode. */ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, enum ufs_dev_pwr_mode pwr_mode) @@ -9576,8 +9507,6 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) req_link_state = UIC_LINK_OFF_STATE; } - ufshpb_suspend(hba); - /* * If we can't transition into any of the low power modes * just gate the clocks. @@ -9731,7 +9660,6 @@ out: ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret); hba->clk_gating.is_suspended = false; ufshcd_release(hba); - ufshpb_resume(hba); } hba->pm_op_in_progress = false; return ret; @@ -9812,7 +9740,6 @@ static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) /* Enable Auto-Hibernate if configured */ ufshcd_auto_hibern8_enable(hba); - ufshpb_resume(hba); goto out; set_old_link_state: @@ -9934,6 +9861,8 @@ out: * * This function will put disable irqs, turn off clocks * and set vreg and hba-vreg in lpm mode. + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_suspend(struct ufs_hba *hba) { @@ -9971,7 +9900,7 @@ static int ufshcd_suspend(struct ufs_hba *hba) * This function basically turns on the regulators, clocks and * irqs of the hba. * - * Returns 0 for success and non-zero for failure + * Return: 0 for success and non-zero for failure. */ static int ufshcd_resume(struct ufs_hba *hba) { @@ -10012,7 +9941,7 @@ out: * Executed before putting the system into a sleep state in which the contents * of main memory are preserved. * - * Returns 0 for success and non-zero for failure + * Return: 0 for success and non-zero for failure. */ int ufshcd_system_suspend(struct device *dev) { @@ -10039,7 +9968,7 @@ EXPORT_SYMBOL(ufshcd_system_suspend); * Executed after waking the system up from a sleep state in which the contents * of main memory were preserved. * - * Returns 0 for success and non-zero for failure + * Return: 0 for success and non-zero for failure. */ int ufshcd_system_resume(struct device *dev) { @@ -10069,7 +9998,7 @@ EXPORT_SYMBOL(ufshcd_system_resume); * * Check the description of ufshcd_suspend() function for more details. * - * Returns 0 for success and non-zero for failure + * Return: 0 for success and non-zero for failure. */ int ufshcd_runtime_suspend(struct device *dev) { @@ -10095,6 +10024,8 @@ EXPORT_SYMBOL(ufshcd_runtime_suspend); * * 1. Turn on all the controller related clocks * 2. Turn ON VCC rail + * + * Return: 0 upon success; < 0 upon failure. */ int ufshcd_runtime_resume(struct device *dev) { @@ -10152,7 +10083,6 @@ void ufshcd_remove(struct ufs_hba *hba) ufshcd_rpm_get_sync(hba); ufs_hwmon_remove(hba); ufs_bsg_remove(hba); - ufshpb_remove(hba); ufs_sysfs_remove_nodes(hba->dev); blk_mq_destroy_queue(hba->tmf_queue); blk_put_queue(hba->tmf_queue); @@ -10230,7 +10160,7 @@ EXPORT_SYMBOL_GPL(ufshcd_dealloc_host); * addressing capability * @hba: per adapter instance * - * Returns 0 for success, non-zero for failure + * Return: 0 for success, non-zero for failure. */ static int ufshcd_set_dma_mask(struct ufs_hba *hba) { @@ -10245,7 +10175,8 @@ static int ufshcd_set_dma_mask(struct ufs_hba *hba) * ufshcd_alloc_host - allocate Host Bus Adapter (HBA) * @dev: pointer to device handle * @hba_handle: driver private handle - * Returns 0 on success, non-zero value on failure + * + * Return: 0 on success, non-zero value on failure. */ int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) { @@ -10301,7 +10232,8 @@ static const struct blk_mq_ops ufshcd_tmf_ops = { * @hba: per-adapter instance * @mmio_base: base register address * @irq: Interrupt line of device - * Returns 0 on success, non-zero value on failure + * + * Return: 0 on success, non-zero value on failure. */ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) { @@ -10632,6 +10564,53 @@ static const struct dev_pm_ops ufshcd_wl_pm_ops = { SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL) }; +static void ufshcd_check_header_layout(void) +{ + /* + * gcc compilers before version 10 cannot do constant-folding for + * sub-byte bitfields. Hence skip the layout checks for gcc 9 and + * before. + */ + if (IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 100000) + return; + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .cci = 3})[0] != 3); + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .ehs_length = 2})[1] != 2); + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .enable_crypto = 1})[2] + != 0x80); + + BUILD_BUG_ON((((u8 *)&(struct request_desc_header){ + .command_type = 5, + .data_direction = 3, + .interrupt = 1, + })[3]) != ((5 << 4) | (3 << 1) | 1)); + + BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ + .dunl = cpu_to_le32(0xdeadbeef)})[1] != + cpu_to_le32(0xdeadbeef)); + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .ocs = 4})[8] != 4); + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .cds = 5})[9] != 5); + + BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ + .dunu = cpu_to_le32(0xbadcafe)})[3] != + cpu_to_le32(0xbadcafe)); + + BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ + .iid = 0xf })[4] != 0xf0); + + BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ + .command_set_type = 0xf })[4] != 0xf); +} + /* * ufs_dev_wlun_template - describes ufs device wlun * ufs-device wlun - used to send pm commands @@ -10657,6 +10636,8 @@ static int __init ufshcd_core_init(void) { int ret; + ufshcd_check_header_layout(); + ufs_debugfs_init(); ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv); diff --git a/drivers/ufs/core/ufshpb.c b/drivers/ufs/core/ufshpb.c deleted file mode 100644 index 255f8b38d0c2..000000000000 --- a/drivers/ufs/core/ufshpb.c +++ /dev/null @@ -1,2668 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Universal Flash Storage Host Performance Booster - * - * Copyright (C) 2017-2021 Samsung Electronics Co., Ltd. - * - * Authors: - * Yongmyung Lee <ymhungry.lee@samsung.com> - * Jinyoung Choi <j-young.choi@samsung.com> - */ - -#include <asm/unaligned.h> -#include <linux/delay.h> -#include <linux/device.h> -#include <linux/module.h> -#include <scsi/scsi_cmnd.h> - -#include "ufshcd-priv.h" -#include "ufshpb.h" -#include "../../scsi/sd.h" - -#define ACTIVATION_THRESHOLD 8 /* 8 IOs */ -#define READ_TO_MS 1000 -#define READ_TO_EXPIRIES 100 -#define POLLING_INTERVAL_MS 200 -#define THROTTLE_MAP_REQ_DEFAULT 1 - -/* memory management */ -static struct kmem_cache *ufshpb_mctx_cache; -static mempool_t *ufshpb_mctx_pool; -static mempool_t *ufshpb_page_pool; -/* A cache size of 2MB can cache ppn in the 1GB range. */ -static unsigned int ufshpb_host_map_kbytes = SZ_2K; -static int tot_active_srgn_pages; - -static struct workqueue_struct *ufshpb_wq; - -static void ufshpb_update_active_info(struct ufshpb_lu *hpb, int rgn_idx, - int srgn_idx); - -bool ufshpb_is_allowed(struct ufs_hba *hba) -{ - return !(hba->ufshpb_dev.hpb_disabled); -} - -/* HPB version 1.0 is called as legacy version. */ -bool ufshpb_is_legacy(struct ufs_hba *hba) -{ - return hba->ufshpb_dev.is_legacy; -} - -static struct ufshpb_lu *ufshpb_get_hpb_data(struct scsi_device *sdev) -{ - return sdev->hostdata; -} - -static int ufshpb_get_state(struct ufshpb_lu *hpb) -{ - return atomic_read(&hpb->hpb_state); -} - -static void ufshpb_set_state(struct ufshpb_lu *hpb, int state) -{ - atomic_set(&hpb->hpb_state, state); -} - -static int ufshpb_is_valid_srgn(struct ufshpb_region *rgn, - struct ufshpb_subregion *srgn) -{ - return rgn->rgn_state != HPB_RGN_INACTIVE && - srgn->srgn_state == HPB_SRGN_VALID; -} - -static bool ufshpb_is_read_cmd(struct scsi_cmnd *cmd) -{ - return req_op(scsi_cmd_to_rq(cmd)) == REQ_OP_READ; -} - -static bool ufshpb_is_write_or_discard(struct scsi_cmnd *cmd) -{ - return op_is_write(req_op(scsi_cmd_to_rq(cmd))) || - op_is_discard(req_op(scsi_cmd_to_rq(cmd))); -} - -static bool ufshpb_is_supported_chunk(struct ufshpb_lu *hpb, int transfer_len) -{ - return transfer_len <= hpb->pre_req_max_tr_len; -} - -static bool ufshpb_is_general_lun(int lun) -{ - return lun < UFS_UPIU_MAX_UNIT_NUM_ID; -} - -static bool ufshpb_is_pinned_region(struct ufshpb_lu *hpb, int rgn_idx) -{ - return hpb->lu_pinned_end != PINNED_NOT_SET && - rgn_idx >= hpb->lu_pinned_start && rgn_idx <= hpb->lu_pinned_end; -} - -static void ufshpb_kick_map_work(struct ufshpb_lu *hpb) -{ - bool ret = false; - unsigned long flags; - - if (ufshpb_get_state(hpb) != HPB_PRESENT) - return; - - spin_lock_irqsave(&hpb->rsp_list_lock, flags); - if (!list_empty(&hpb->lh_inact_rgn) || !list_empty(&hpb->lh_act_srgn)) - ret = true; - spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); - - if (ret) - queue_work(ufshpb_wq, &hpb->map_work); -} - -static bool ufshpb_is_hpb_rsp_valid(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp, - struct utp_hpb_rsp *rsp_field) -{ - /* Check HPB_UPDATE_ALERT */ - if (!(lrbp->ucd_rsp_ptr->header.dword_2 & - UPIU_HEADER_DWORD(0, 2, 0, 0))) - return false; - - if (be16_to_cpu(rsp_field->sense_data_len) != DEV_SENSE_SEG_LEN || - rsp_field->desc_type != DEV_DES_TYPE || - rsp_field->additional_len != DEV_ADDITIONAL_LEN || - rsp_field->active_rgn_cnt > MAX_ACTIVE_NUM || - rsp_field->inactive_rgn_cnt > MAX_INACTIVE_NUM || - rsp_field->hpb_op == HPB_RSP_NONE || - (rsp_field->hpb_op == HPB_RSP_REQ_REGION_UPDATE && - !rsp_field->active_rgn_cnt && !rsp_field->inactive_rgn_cnt)) - return false; - - if (!ufshpb_is_general_lun(rsp_field->lun)) { - dev_warn(hba->dev, "ufshpb: lun(%d) not supported\n", - lrbp->lun); - return false; - } - - return true; -} - -static void ufshpb_iterate_rgn(struct ufshpb_lu *hpb, int rgn_idx, int srgn_idx, - int srgn_offset, int cnt, bool set_dirty) -{ - struct ufshpb_region *rgn; - struct ufshpb_subregion *srgn, *prev_srgn = NULL; - int set_bit_len; - int bitmap_len; - unsigned long flags; - -next_srgn: - rgn = hpb->rgn_tbl + rgn_idx; - srgn = rgn->srgn_tbl + srgn_idx; - - if (likely(!srgn->is_last)) - bitmap_len = hpb->entries_per_srgn; - else - bitmap_len = hpb->last_srgn_entries; - - if ((srgn_offset + cnt) > bitmap_len) - set_bit_len = bitmap_len - srgn_offset; - else - set_bit_len = cnt; - - spin_lock_irqsave(&hpb->rgn_state_lock, flags); - if (rgn->rgn_state != HPB_RGN_INACTIVE) { - if (set_dirty) { - if (srgn->srgn_state == HPB_SRGN_VALID) - bitmap_set(srgn->mctx->ppn_dirty, srgn_offset, - set_bit_len); - } else if (hpb->is_hcm) { - /* rewind the read timer for lru regions */ - rgn->read_timeout = ktime_add_ms(ktime_get(), - rgn->hpb->params.read_timeout_ms); - rgn->read_timeout_expiries = - rgn->hpb->params.read_timeout_expiries; - } - } - spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); - - if (hpb->is_hcm && prev_srgn != srgn) { - bool activate = false; - - spin_lock(&rgn->rgn_lock); - if (set_dirty) { - rgn->reads -= srgn->reads; - srgn->reads = 0; - set_bit(RGN_FLAG_DIRTY, &rgn->rgn_flags); - } else { - srgn->reads++; - rgn->reads++; - if (srgn->reads == hpb->params.activation_thld) - activate = true; - } - spin_unlock(&rgn->rgn_lock); - - if (activate || - test_and_clear_bit(RGN_FLAG_UPDATE, &rgn->rgn_flags)) { - spin_lock_irqsave(&hpb->rsp_list_lock, flags); - ufshpb_update_active_info(hpb, rgn_idx, srgn_idx); - spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); - dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, - "activate region %d-%d\n", rgn_idx, srgn_idx); - } - - prev_srgn = srgn; - } - - srgn_offset = 0; - if (++srgn_idx == hpb->srgns_per_rgn) { - srgn_idx = 0; - rgn_idx++; - } - - cnt -= set_bit_len; - if (cnt > 0) - goto next_srgn; -} - -static bool ufshpb_test_ppn_dirty(struct ufshpb_lu *hpb, int rgn_idx, - int srgn_idx, int srgn_offset, int cnt) -{ - struct ufshpb_region *rgn; - struct ufshpb_subregion *srgn; - int bitmap_len; - int bit_len; - -next_srgn: - rgn = hpb->rgn_tbl + rgn_idx; - srgn = rgn->srgn_tbl + srgn_idx; - - if (!ufshpb_is_valid_srgn(rgn, srgn)) - return true; - - /* - * If the region state is active, mctx must be allocated. - * In this case, check whether the region is evicted or - * mctx allocation fail. - */ - if (unlikely(!srgn->mctx)) { - dev_err(&hpb->sdev_ufs_lu->sdev_dev, - "no mctx in region %d subregion %d.\n", - srgn->rgn_idx, srgn->srgn_idx); - return true; - } - - if (likely(!srgn->is_last)) - bitmap_len = hpb->entries_per_srgn; - else - bitmap_len = hpb->last_srgn_entries; - - if ((srgn_offset + cnt) > bitmap_len) - bit_len = bitmap_len - srgn_offset; - else - bit_len = cnt; - - if (find_next_bit(srgn->mctx->ppn_dirty, bit_len + srgn_offset, - srgn_offset) < bit_len + srgn_offset) - return true; - - srgn_offset = 0; - if (++srgn_idx == hpb->srgns_per_rgn) { - srgn_idx = 0; - rgn_idx++; - } - - cnt -= bit_len; - if (cnt > 0) - goto next_srgn; - - return false; -} - -static inline bool is_rgn_dirty(struct ufshpb_region *rgn) -{ - return test_bit(RGN_FLAG_DIRTY, &rgn->rgn_flags); -} - -static int ufshpb_fill_ppn_from_page(struct ufshpb_lu *hpb, - struct ufshpb_map_ctx *mctx, int pos, - int len, __be64 *ppn_buf) -{ - struct page *page; - int index, offset; - int copied; - - index = pos / (PAGE_SIZE / HPB_ENTRY_SIZE); - offset = pos % (PAGE_SIZE / HPB_ENTRY_SIZE); - - if ((offset + len) <= (PAGE_SIZE / HPB_ENTRY_SIZE)) - copied = len; - else - copied = (PAGE_SIZE / HPB_ENTRY_SIZE) - offset; - - page = mctx->m_page[index]; - if (unlikely(!page)) { - dev_err(&hpb->sdev_ufs_lu->sdev_dev, - "error. cannot find page in mctx\n"); - return -ENOMEM; - } - - memcpy(ppn_buf, page_address(page) + (offset * HPB_ENTRY_SIZE), - copied * HPB_ENTRY_SIZE); - - return copied; -} - -static void -ufshpb_get_pos_from_lpn(struct ufshpb_lu *hpb, unsigned long lpn, int *rgn_idx, - int *srgn_idx, int *offset) -{ - int rgn_offset; - - *rgn_idx = lpn >> hpb->entries_per_rgn_shift; - rgn_offset = lpn & hpb->entries_per_rgn_mask; - *srgn_idx = rgn_offset >> hpb->entries_per_srgn_shift; - *offset = rgn_offset & hpb->entries_per_srgn_mask; -} - -static void -ufshpb_set_hpb_read_to_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, - __be64 ppn, u8 transfer_len) -{ - unsigned char *cdb = lrbp->cmd->cmnd; - __be64 ppn_tmp = ppn; - cdb[0] = UFSHPB_READ; - - if (hba->dev_quirks & UFS_DEVICE_QUIRK_SWAP_L2P_ENTRY_FOR_HPB_READ) - ppn_tmp = (__force __be64)swab64((__force u64)ppn); - - /* ppn value is stored as big-endian in the host memory */ - memcpy(&cdb[6], &ppn_tmp, sizeof(__be64)); - cdb[14] = transfer_len; - cdb[15] = 0; - - lrbp->cmd->cmd_len = UFS_CDB_SIZE; -} - -/* - * This function will set up HPB read command using host-side L2P map data. - */ -int ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) -{ - struct ufshpb_lu *hpb; - struct ufshpb_region *rgn; - struct ufshpb_subregion *srgn; - struct scsi_cmnd *cmd = lrbp->cmd; - u32 lpn; - __be64 ppn; - unsigned long flags; - int transfer_len, rgn_idx, srgn_idx, srgn_offset; - int err = 0; - - hpb = ufshpb_get_hpb_data(cmd->device); - if (!hpb) - return -ENODEV; - - if (ufshpb_get_state(hpb) == HPB_INIT) - return -ENODEV; - - if (ufshpb_get_state(hpb) != HPB_PRESENT) { - dev_notice(&hpb->sdev_ufs_lu->sdev_dev, - "%s: ufshpb state is not PRESENT", __func__); - return -ENODEV; - } - - if (blk_rq_is_passthrough(scsi_cmd_to_rq(cmd)) || - (!ufshpb_is_write_or_discard(cmd) && - !ufshpb_is_read_cmd(cmd))) - return 0; - - transfer_len = sectors_to_logical(cmd->device, - blk_rq_sectors(scsi_cmd_to_rq(cmd))); - if (unlikely(!transfer_len)) - return 0; - - lpn = sectors_to_logical(cmd->device, blk_rq_pos(scsi_cmd_to_rq(cmd))); - ufshpb_get_pos_from_lpn(hpb, lpn, &rgn_idx, &srgn_idx, &srgn_offset); - rgn = hpb->rgn_tbl + rgn_idx; - srgn = rgn->srgn_tbl + srgn_idx; - - /* If command type is WRITE or DISCARD, set bitmap as dirty */ - if (ufshpb_is_write_or_discard(cmd)) { - ufshpb_iterate_rgn(hpb, rgn_idx, srgn_idx, srgn_offset, - transfer_len, true); - return 0; - } - - if (!ufshpb_is_supported_chunk(hpb, transfer_len)) - return 0; - - if (hpb->is_hcm) { - /* - * in host control mode, reads are the main source for - * activation trials. - */ - ufshpb_iterate_rgn(hpb, rgn_idx, srgn_idx, srgn_offset, - transfer_len, false); - - /* keep those counters normalized */ - if (rgn->reads > hpb->entries_per_srgn) - schedule_work(&hpb->ufshpb_normalization_work); - } - - spin_lock_irqsave(&hpb->rgn_state_lock, flags); - if (ufshpb_test_ppn_dirty(hpb, rgn_idx, srgn_idx, srgn_offset, - transfer_len)) { - hpb->stats.miss_cnt++; - spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); - return 0; - } - - err = ufshpb_fill_ppn_from_page(hpb, srgn->mctx, srgn_offset, 1, &ppn); - spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); - if (unlikely(err < 0)) { - /* - * In this case, the region state is active, - * but the ppn table is not allocated. - * Make sure that ppn table must be allocated on - * active state. - */ - dev_err(hba->dev, "get ppn failed. err %d\n", err); - return err; - } - - ufshpb_set_hpb_read_to_upiu(hba, lrbp, ppn, transfer_len); - - hpb->stats.hit_cnt++; - return 0; -} - -static struct ufshpb_req *ufshpb_get_req(struct ufshpb_lu *hpb, int rgn_idx, - enum req_op op, bool atomic) -{ - struct ufshpb_req *rq; - struct request *req; - int retries = HPB_MAP_REQ_RETRIES; - - rq = kmem_cache_alloc(hpb->map_req_cache, GFP_KERNEL); - if (!rq) - return NULL; - -retry: - req = blk_mq_alloc_request(hpb->sdev_ufs_lu->request_queue, op, - BLK_MQ_REQ_NOWAIT); - - if (!atomic && (PTR_ERR(req) == -EWOULDBLOCK) && (--retries > 0)) { - usleep_range(3000, 3100); - goto retry; - } - - if (IS_ERR(req)) - goto free_rq; - - rq->hpb = hpb; - rq->req = req; - rq->rb.rgn_idx = rgn_idx; - - return rq; - -free_rq: - kmem_cache_free(hpb->map_req_cache, rq); - return NULL; -} - -static void ufshpb_put_req(struct ufshpb_lu *hpb, struct ufshpb_req *rq) -{ - blk_mq_free_request(rq->req); - kmem_cache_free(hpb->map_req_cache, rq); -} - -static struct ufshpb_req *ufshpb_get_map_req(struct ufshpb_lu *hpb, - struct ufshpb_subregion *srgn) -{ - struct ufshpb_req *map_req; - struct bio *bio; - unsigned long flags; - - if (hpb->is_hcm && - hpb->num_inflight_map_req >= hpb->params.inflight_map_req) { - dev_info(&hpb->sdev_ufs_lu->sdev_dev, - "map_req throttle. inflight %d throttle %d", - hpb->num_inflight_map_req, - hpb->params.inflight_map_req); - return NULL; - } - - map_req = ufshpb_get_req(hpb, srgn->rgn_idx, REQ_OP_DRV_IN, false); - if (!map_req) - return NULL; - - bio = bio_alloc(NULL, hpb->pages_per_srgn, 0, GFP_KERNEL); - if (!bio) { - ufshpb_put_req(hpb, map_req); - return NULL; - } - - map_req->bio = bio; - - map_req->rb.srgn_idx = srgn->srgn_idx; - map_req->rb.mctx = srgn->mctx; - - spin_lock_irqsave(&hpb->param_lock, flags); - hpb->num_inflight_map_req++; - spin_unlock_irqrestore(&hpb->param_lock, flags); - - return map_req; -} - -static void ufshpb_put_map_req(struct ufshpb_lu *hpb, - struct ufshpb_req *map_req) -{ - unsigned long flags; - - bio_put(map_req->bio); - ufshpb_put_req(hpb, map_req); - - spin_lock_irqsave(&hpb->param_lock, flags); - hpb->num_inflight_map_req--; - spin_unlock_irqrestore(&hpb->param_lock, flags); -} - -static int ufshpb_clear_dirty_bitmap(struct ufshpb_lu *hpb, - struct ufshpb_subregion *srgn) -{ - struct ufshpb_region *rgn; - u32 num_entries = hpb->entries_per_srgn; - - if (!srgn->mctx) { - dev_err(&hpb->sdev_ufs_lu->sdev_dev, - "no mctx in region %d subregion %d.\n", - srgn->rgn_idx, srgn->srgn_idx); - return -1; - } - - if (unlikely(srgn->is_last)) - num_entries = hpb->last_srgn_entries; - - bitmap_zero(srgn->mctx->ppn_dirty, num_entries); - - rgn = hpb->rgn_tbl + srgn->rgn_idx; - clear_bit(RGN_FLAG_DIRTY, &rgn->rgn_flags); - - return 0; -} - -static void ufshpb_update_active_info(struct ufshpb_lu *hpb, int rgn_idx, - int srgn_idx) -{ - struct ufshpb_region *rgn; - struct ufshpb_subregion *srgn; - - rgn = hpb->rgn_tbl + rgn_idx; - srgn = rgn->srgn_tbl + srgn_idx; - - list_del_init(&rgn->list_inact_rgn); - - if (list_empty(&srgn->list_act_srgn)) - list_add_tail(&srgn->list_act_srgn, &hpb->lh_act_srgn); - - hpb->stats.rcmd_active_cnt++; -} - -static void ufshpb_update_inactive_info(struct ufshpb_lu *hpb, int rgn_idx) -{ - struct ufshpb_region *rgn; - struct ufshpb_subregion *srgn; - int srgn_idx; - - rgn = hpb->rgn_tbl + rgn_idx; - - for_each_sub_region(rgn, srgn_idx, srgn) - list_del_init(&srgn->list_act_srgn); - - if (list_empty(&rgn->list_inact_rgn)) - list_add_tail(&rgn->list_inact_rgn, &hpb->lh_inact_rgn); - - hpb->stats.rcmd_inactive_cnt++; -} - -static void ufshpb_activate_subregion(struct ufshpb_lu *hpb, - struct ufshpb_subregion *srgn) -{ - struct ufshpb_region *rgn; - - /* - * If there is no mctx in subregion - * after I/O progress for HPB_READ_BUFFER, the region to which the - * subregion belongs was evicted. - * Make sure the region must not evict in I/O progress - */ - if (!srgn->mctx) { - dev_err(&hpb->sdev_ufs_lu->sdev_dev, - "no mctx in region %d subregion %d.\n", - srgn->rgn_idx, srgn->srgn_idx); - srgn->srgn_state = HPB_SRGN_INVALID; - return; - } - - rgn = hpb->rgn_tbl + srgn->rgn_idx; - - if (unlikely(rgn->rgn_state == HPB_RGN_INACTIVE)) { - dev_err(&hpb->sdev_ufs_lu->sdev_dev, - "region %d subregion %d evicted\n", - srgn->rgn_idx, srgn->srgn_idx); - srgn->srgn_state = HPB_SRGN_INVALID; - return; - } - srgn->srgn_state = HPB_SRGN_VALID; -} - -static enum rq_end_io_ret ufshpb_umap_req_compl_fn(struct request *req, - blk_status_t error) -{ - struct ufshpb_req *umap_req = req->end_io_data; - - ufshpb_put_req(umap_req->hpb, umap_req); - return RQ_END_IO_NONE; -} - -static enum rq_end_io_ret ufshpb_map_req_compl_fn(struct request *req, - blk_status_t error) -{ - struct ufshpb_req *map_req = req->end_io_data; - struct ufshpb_lu *hpb = map_req->hpb; - struct ufshpb_subregion *srgn; - unsigned long flags; - - srgn = hpb->rgn_tbl[map_req->rb.rgn_idx].srgn_tbl + - map_req->rb.srgn_idx; - - ufshpb_clear_dirty_bitmap(hpb, srgn); - spin_lock_irqsave(&hpb->rgn_state_lock, flags); - ufshpb_activate_subregion(hpb, srgn); - spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); - - ufshpb_put_map_req(map_req->hpb, map_req); - return RQ_END_IO_NONE; -} - -static void ufshpb_set_unmap_cmd(unsigned char *cdb, struct ufshpb_region *rgn) -{ - cdb[0] = UFSHPB_WRITE_BUFFER; - cdb[1] = rgn ? UFSHPB_WRITE_BUFFER_INACT_SINGLE_ID : - UFSHPB_WRITE_BUFFER_INACT_ALL_ID; - if (rgn) - put_unaligned_be16(rgn->rgn_idx, &cdb[2]); - cdb[9] = 0x00; -} - -static void ufshpb_set_read_buf_cmd(unsigned char *cdb, int rgn_idx, - int srgn_idx, int srgn_mem_size) -{ - cdb[0] = UFSHPB_READ_BUFFER; - cdb[1] = UFSHPB_READ_BUFFER_ID; - - put_unaligned_be16(rgn_idx, &cdb[2]); - put_unaligned_be16(srgn_idx, &cdb[4]); - put_unaligned_be24(srgn_mem_size, &cdb[6]); - - cdb[9] = 0x00; -} - -static void ufshpb_execute_umap_req(struct ufshpb_lu *hpb, - struct ufshpb_req *umap_req, - struct ufshpb_region *rgn) -{ - struct request *req = umap_req->req; - struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(req); - - req->timeout = 0; - req->end_io_data = umap_req; - req->end_io = ufshpb_umap_req_compl_fn; - - ufshpb_set_unmap_cmd(scmd->cmnd, rgn); - scmd->cmd_len = HPB_WRITE_BUFFER_CMD_LENGTH; - - blk_execute_rq_nowait(req, true); - - hpb->stats.umap_req_cnt++; -} - -static int ufshpb_execute_map_req(struct ufshpb_lu *hpb, - struct ufshpb_req *map_req, bool last) -{ - struct request_queue *q; - struct request *req; - struct scsi_cmnd *scmd; - int mem_size = hpb->srgn_mem_size; - int ret = 0; - int i; - - q = hpb->sdev_ufs_lu->request_queue; - for (i = 0; i < hpb->pages_per_srgn; i++) { - ret = bio_add_pc_page(q, map_req->bio, map_req->rb.mctx->m_page[i], - PAGE_SIZE, 0); - if (ret != PAGE_SIZE) { - dev_err(&hpb->sdev_ufs_lu->sdev_dev, - "bio_add_pc_page fail %d - %d\n", - map_req->rb.rgn_idx, map_req->rb.srgn_idx); - return ret; - } - } - - req = map_req->req; - - blk_rq_append_bio(req, map_req->bio); - - req->end_io_data = map_req; - req->end_io = ufshpb_map_req_compl_fn; - - if (unlikely(last)) - mem_size = hpb->last_srgn_entries * HPB_ENTRY_SIZE; - - scmd = blk_mq_rq_to_pdu(req); - ufshpb_set_read_buf_cmd(scmd->cmnd, map_req->rb.rgn_idx, - map_req->rb.srgn_idx, mem_size); - scmd->cmd_len = HPB_READ_BUFFER_CMD_LENGTH; - - blk_execute_rq_nowait(req, true); - - hpb->stats.map_req_cnt++; - return 0; -} - -static struct ufshpb_map_ctx *ufshpb_get_map_ctx(struct ufshpb_lu *hpb, - bool last) -{ - struct ufshpb_map_ctx *mctx; - u32 num_entries = hpb->entries_per_srgn; - int i, j; - - mctx = mempool_alloc(ufshpb_mctx_pool, GFP_KERNEL); - if (!mctx) - return NULL; - - mctx->m_page = kmem_cache_alloc(hpb->m_page_cache, GFP_KERNEL); - if (!mctx->m_page) - goto release_mctx; - - if (unlikely(last)) - num_entries = hpb->last_srgn_entries; - - mctx->ppn_dirty = bitmap_zalloc(num_entries, GFP_KERNEL); - if (!mctx->ppn_dirty) - goto release_m_page; - - for (i = 0; i < hpb->pages_per_srgn; i++) { - mctx->m_page[i] = mempool_alloc(ufshpb_page_pool, GFP_KERNEL); - if (!mctx->m_page[i]) { - for (j = 0; j < i; j++) - mempool_free(mctx->m_page[j], ufshpb_page_pool); - goto release_ppn_dirty; - } - clear_page(page_address(mctx->m_page[i])); - } - - return mctx; - -release_ppn_dirty: - bitmap_free(mctx->ppn_dirty); -release_m_page: - kmem_cache_free(hpb->m_page_cache, mctx->m_page); -release_mctx: - mempool_free(mctx, ufshpb_mctx_pool); - return NULL; -} - -static void ufshpb_put_map_ctx(struct ufshpb_lu *hpb, - struct ufshpb_map_ctx *mctx) -{ - int i; - - for (i = 0; i < hpb->pages_per_srgn; i++) - mempool_free(mctx->m_page[i], ufshpb_page_pool); - - bitmap_free(mctx->ppn_dirty); - kmem_cache_free(hpb->m_page_cache, mctx->m_page); - mempool_free(mctx, ufshpb_mctx_pool); -} - -static int ufshpb_check_srgns_issue_state(struct ufshpb_lu *hpb, - struct ufshpb_region *rgn) -{ - struct ufshpb_subregion *srgn; - int srgn_idx; - - for_each_sub_region(rgn, srgn_idx, srgn) - if (srgn->srgn_state == HPB_SRGN_ISSUED) - return -EPERM; - - return 0; -} - -static void ufshpb_read_to_handler(struct work_struct *work) -{ - struct ufshpb_lu *hpb = container_of(work, struct ufshpb_lu, - ufshpb_read_to_work.work); - struct victim_select_info *lru_info = &hpb->lru_info; - struct ufshpb_region *rgn, *next_rgn; - unsigned long flags; - unsigned int poll; - LIST_HEAD(expired_list); - - if (test_and_set_bit(TIMEOUT_WORK_RUNNING, &hpb->work_data_bits)) - return; - - spin_lock_irqsave(&hpb->rgn_state_lock, flags); - - list_for_each_entry_safe(rgn, next_rgn, &lru_info->lh_lru_rgn, - list_lru_rgn) { - bool timedout = ktime_after(ktime_get(), rgn->read_timeout); - - if (timedout) { - rgn->read_timeout_expiries--; - if (is_rgn_dirty(rgn) || - rgn->read_timeout_expiries == 0) - list_add(&rgn->list_expired_rgn, &expired_list); - else - rgn->read_timeout = ktime_add_ms(ktime_get(), - hpb->params.read_timeout_ms); - } - } - - spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); - - list_for_each_entry_safe(rgn, next_rgn, &expired_list, - list_expired_rgn) { - list_del_init(&rgn->list_expired_rgn); - spin_lock_irqsave(&hpb->rsp_list_lock, flags); - ufshpb_update_inactive_info(hpb, rgn->rgn_idx); - spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); - } - - ufshpb_kick_map_work(hpb); - - clear_bit(TIMEOUT_WORK_RUNNING, &hpb->work_data_bits); - - poll = hpb->params.timeout_polling_interval_ms; - schedule_delayed_work(&hpb->ufshpb_read_to_work, - msecs_to_jiffies(poll)); -} - -static void ufshpb_add_lru_info(struct victim_select_info *lru_info, - struct ufshpb_region *rgn) -{ - rgn->rgn_state = HPB_RGN_ACTIVE; - list_add_tail(&rgn->list_lru_rgn, &lru_info->lh_lru_rgn); - atomic_inc(&lru_info->active_cnt); - if (rgn->hpb->is_hcm) { - rgn->read_timeout = - ktime_add_ms(ktime_get(), - rgn->hpb->params.read_timeout_ms); - rgn->read_timeout_expiries = - rgn->hpb->params.read_timeout_expiries; - } -} - -static void ufshpb_hit_lru_info(struct victim_select_info *lru_info, - struct ufshpb_region *rgn) -{ - list_move_tail(&rgn->list_lru_rgn, &lru_info->lh_lru_rgn); -} - -static struct ufshpb_region *ufshpb_victim_lru_info(struct ufshpb_lu *hpb) -{ - struct victim_select_info *lru_info = &hpb->lru_info; - struct ufshpb_region *rgn, *victim_rgn = NULL; - - list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) { - if (ufshpb_check_srgns_issue_state(hpb, rgn)) - continue; - - /* - * in host control mode, verify that the exiting region - * has fewer reads - */ - if (hpb->is_hcm && - rgn->reads > hpb->params.eviction_thld_exit) - continue; - - victim_rgn = rgn; - break; - } - - if (!victim_rgn) - dev_err(&hpb->sdev_ufs_lu->sdev_dev, - "%s: no region allocated\n", - __func__); - - return victim_rgn; -} - -static void ufshpb_cleanup_lru_info(struct victim_select_info *lru_info, - struct ufshpb_region *rgn) -{ - list_del_init(&rgn->list_lru_rgn); - rgn->rgn_state = HPB_RGN_INACTIVE; - atomic_dec(&lru_info->active_cnt); -} - -static void ufshpb_purge_active_subregion(struct ufshpb_lu *hpb, - struct ufshpb_subregion *srgn) -{ - if (srgn->srgn_state != HPB_SRGN_UNUSED) { - ufshpb_put_map_ctx(hpb, srgn->mctx); - srgn->srgn_state = HPB_SRGN_UNUSED; - srgn->mctx = NULL; - } -} - -static int ufshpb_issue_umap_req(struct ufshpb_lu *hpb, - struct ufshpb_region *rgn, - bool atomic) -{ - struct ufshpb_req *umap_req; - int rgn_idx = rgn ? rgn->rgn_idx : 0; - - umap_req = ufshpb_get_req(hpb, rgn_idx, REQ_OP_DRV_OUT, atomic); - if (!umap_req) - return -ENOMEM; - - ufshpb_execute_umap_req(hpb, umap_req, rgn); - - return 0; -} - -static int ufshpb_issue_umap_single_req(struct ufshpb_lu *hpb, - struct ufshpb_region *rgn) -{ - return ufshpb_issue_umap_req(hpb, rgn, true); -} - -static void __ufshpb_evict_region(struct ufshpb_lu *hpb, - struct ufshpb_region *rgn) -{ - struct victim_select_info *lru_info; - struct ufshpb_subregion *srgn; - int srgn_idx; - - lru_info = &hpb->lru_info; - - dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, "evict region %d\n", rgn->rgn_idx); - - ufshpb_cleanup_lru_info(lru_info, rgn); - - for_each_sub_region(rgn, srgn_idx, srgn) - ufshpb_purge_active_subregion(hpb, srgn); -} - -static int ufshpb_evict_region(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) -{ - unsigned long flags; - int ret = 0; - - spin_lock_irqsave(&hpb->rgn_state_lock, flags); - if (rgn->rgn_state == HPB_RGN_PINNED) { - dev_warn(&hpb->sdev_ufs_lu->sdev_dev, - "pinned region cannot drop-out. region %d\n", - rgn->rgn_idx); - goto out; - } - - if (!list_empty(&rgn->list_lru_rgn)) { - if (ufshpb_check_srgns_issue_state(hpb, rgn)) { - ret = -EBUSY; - goto out; - } - - if (hpb->is_hcm) { - spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); - ret = ufshpb_issue_umap_single_req(hpb, rgn); - spin_lock_irqsave(&hpb->rgn_state_lock, flags); - if (ret) - goto out; - } - - __ufshpb_evict_region(hpb, rgn); - } -out: - spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); - return ret; -} - -static int ufshpb_issue_map_req(struct ufshpb_lu *hpb, - struct ufshpb_region *rgn, - struct ufshpb_subregion *srgn) -{ - struct ufshpb_req *map_req; - unsigned long flags; - int ret; - int err = -EAGAIN; - bool alloc_required = false; - enum HPB_SRGN_STATE state = HPB_SRGN_INVALID; - - spin_lock_irqsave(&hpb->rgn_state_lock, flags); - - if (ufshpb_get_state(hpb) != HPB_PRESENT) { - dev_notice(&hpb->sdev_ufs_lu->sdev_dev, - "%s: ufshpb state is not PRESENT\n", __func__); - goto unlock_out; - } - - if ((rgn->rgn_state == HPB_RGN_INACTIVE) && - (srgn->srgn_state == HPB_SRGN_INVALID)) { - err = 0; - goto unlock_out; - } - - if (srgn->srgn_state == HPB_SRGN_UNUSED) - alloc_required = true; - - /* - * If the subregion is already ISSUED state, - * a specific event (e.g., GC or wear-leveling, etc.) occurs in - * the device and HPB response for map loading is received. - * In this case, after finishing the HPB_READ_BUFFER, - * the next HPB_READ_BUFFER is performed again to obtain the latest - * map data. - */ - if (srgn->srgn_state == HPB_SRGN_ISSUED) - goto unlock_out; - - srgn->srgn_state = HPB_SRGN_ISSUED; - spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); - - if (alloc_required) { - srgn->mctx = ufshpb_get_map_ctx(hpb, srgn->is_last); - if (!srgn->mctx) { - dev_err(&hpb->sdev_ufs_lu->sdev_dev, - "get map_ctx failed. region %d - %d\n", - rgn->rgn_idx, srgn->srgn_idx); - state = HPB_SRGN_UNUSED; - goto change_srgn_state; - } - } - - map_req = ufshpb_get_map_req(hpb, srgn); - if (!map_req) - goto change_srgn_state; - - - ret = ufshpb_execute_map_req(hpb, map_req, srgn->is_last); - if (ret) { - dev_err(&hpb->sdev_ufs_lu->sdev_dev, - "%s: issue map_req failed: %d, region %d - %d\n", - __func__, ret, srgn->rgn_idx, srgn->srgn_idx); - goto free_map_req; - } - return 0; - -free_map_req: - ufshpb_put_map_req(hpb, map_req); -change_srgn_state: - spin_lock_irqsave(&hpb->rgn_state_lock, flags); - srgn->srgn_state = state; -unlock_out: - spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); - return err; -} - -static int ufshpb_add_region(struct ufshpb_lu *hpb, struct ufshpb_region *rgn) -{ - struct ufshpb_region *victim_rgn = NULL; - struct victim_select_info *lru_info = &hpb->lru_info; - unsigned long flags; - int ret = 0; - - spin_lock_irqsave(&hpb->rgn_state_lock, flags); - /* - * If region belongs to lru_list, just move the region - * to the front of lru list because the state of the region - * is already active-state. - */ - if (!list_empty(&rgn->list_lru_rgn)) { - ufshpb_hit_lru_info(lru_info, rgn); - goto out; - } - - if (rgn->rgn_state == HPB_RGN_INACTIVE) { - if (atomic_read(&lru_info->active_cnt) == - lru_info->max_lru_active_cnt) { - /* - * If the maximum number of active regions - * is exceeded, evict the least recently used region. - * This case may occur when the device responds - * to the eviction information late. - * It is okay to evict the least recently used region, - * because the device could detect this region - * by not issuing HPB_READ - * - * in host control mode, verify that the entering - * region has enough reads - */ - if (hpb->is_hcm && - rgn->reads < hpb->params.eviction_thld_enter) { - ret = -EACCES; - goto out; - } - - victim_rgn = ufshpb_victim_lru_info(hpb); - if (!victim_rgn) { - dev_warn(&hpb->sdev_ufs_lu->sdev_dev, - "cannot get victim region %s\n", - hpb->is_hcm ? "" : "error"); - ret = -ENOMEM; - goto out; - } - - dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, - "LRU full (%d), choose victim %d\n", - atomic_read(&lru_info->active_cnt), - victim_rgn->rgn_idx); - - if (hpb->is_hcm) { - spin_unlock_irqrestore(&hpb->rgn_state_lock, - flags); - ret = ufshpb_issue_umap_single_req(hpb, - victim_rgn); - spin_lock_irqsave(&hpb->rgn_state_lock, - flags); - if (ret) - goto out; - } - - __ufshpb_evict_region(hpb, victim_rgn); - } - - /* - * When a region is added to lru_info list_head, - * it is guaranteed that the subregion has been - * assigned all mctx. If failed, try to receive mctx again - * without being added to lru_info list_head - */ - ufshpb_add_lru_info(lru_info, rgn); - } -out: - spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); - return ret; -} -/** - *ufshpb_submit_region_inactive() - submit a region to be inactivated later - *@hpb: per-LU HPB instance - *@region_index: the index associated with the region that will be inactivated later - */ -static void ufshpb_submit_region_inactive(struct ufshpb_lu *hpb, int region_index) -{ - int subregion_index; - struct ufshpb_region *rgn; - struct ufshpb_subregion *srgn; - - /* - * Remove this region from active region list and add it to inactive list - */ - spin_lock(&hpb->rsp_list_lock); - ufshpb_update_inactive_info(hpb, region_index); - spin_unlock(&hpb->rsp_list_lock); - - rgn = hpb->rgn_tbl + region_index; - - /* - * Set subregion state to be HPB_SRGN_INVALID, there will no HPB read on this subregion - */ - spin_lock(&hpb->rgn_state_lock); - if (rgn->rgn_state != HPB_RGN_INACTIVE) { - for (subregion_index = 0; subregion_index < rgn->srgn_cnt; subregion_index++) { - srgn = rgn->srgn_tbl + subregion_index; - if (srgn->srgn_state == HPB_SRGN_VALID) - srgn->srgn_state = HPB_SRGN_INVALID; - } - } - spin_unlock(&hpb->rgn_state_lock); -} - -static void ufshpb_rsp_req_region_update(struct ufshpb_lu *hpb, - struct utp_hpb_rsp *rsp_field) -{ - struct ufshpb_region *rgn; - struct ufshpb_subregion *srgn; - int i, rgn_i, srgn_i; - - BUILD_BUG_ON(sizeof(struct ufshpb_active_field) != HPB_ACT_FIELD_SIZE); - /* - * If the active region and the inactive region are the same, - * we will inactivate this region. - * The device could check this (region inactivated) and - * will response the proper active region information - */ - for (i = 0; i < rsp_field->active_rgn_cnt; i++) { - rgn_i = - be16_to_cpu(rsp_field->hpb_active_field[i].active_rgn); - srgn_i = - be16_to_cpu(rsp_field->hpb_active_field[i].active_srgn); - - rgn = hpb->rgn_tbl + rgn_i; - if (hpb->is_hcm && - (rgn->rgn_state != HPB_RGN_ACTIVE || is_rgn_dirty(rgn))) { - /* - * in host control mode, subregion activation - * recommendations are only allowed to active regions. - * Also, ignore recommendations for dirty regions - the - * host will make decisions concerning those by himself - */ - continue; - } - - dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, - "activate(%d) region %d - %d\n", i, rgn_i, srgn_i); - - spin_lock(&hpb->rsp_list_lock); - ufshpb_update_active_info(hpb, rgn_i, srgn_i); - spin_unlock(&hpb->rsp_list_lock); - - srgn = rgn->srgn_tbl + srgn_i; - - /* blocking HPB_READ */ - spin_lock(&hpb->rgn_state_lock); - if (srgn->srgn_state == HPB_SRGN_VALID) - srgn->srgn_state = HPB_SRGN_INVALID; - spin_unlock(&hpb->rgn_state_lock); - } - - if (hpb->is_hcm) { - /* - * in host control mode the device is not allowed to inactivate - * regions - */ - goto out; - } - - for (i = 0; i < rsp_field->inactive_rgn_cnt; i++) { - rgn_i = be16_to_cpu(rsp_field->hpb_inactive_field[i]); - dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, "inactivate(%d) region %d\n", i, rgn_i); - ufshpb_submit_region_inactive(hpb, rgn_i); - } - -out: - dev_dbg(&hpb->sdev_ufs_lu->sdev_dev, "Noti: #ACT %u #INACT %u\n", - rsp_field->active_rgn_cnt, rsp_field->inactive_rgn_cnt); - - if (ufshpb_get_state(hpb) == HPB_PRESENT) - queue_work(ufshpb_wq, &hpb->map_work); -} - -/* - * Set the flags of all active regions to RGN_FLAG_UPDATE to let host side reload L2P entries later - */ -static void ufshpb_set_regions_update(struct ufshpb_lu *hpb) -{ - struct victim_select_info *lru_info = &hpb->lru_info; - struct ufshpb_region *rgn; - unsigned long flags; - - spin_lock_irqsave(&hpb->rgn_state_lock, flags); - - list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) - set_bit(RGN_FLAG_UPDATE, &rgn->rgn_flags); - - spin_unlock_irqrestore(&hpb->rgn_state_lock, flags); -} - -static void ufshpb_dev_reset_handler(struct ufs_hba *hba) -{ - struct scsi_device *sdev; - struct ufshpb_lu *hpb; - - __shost_for_each_device(sdev, hba->host) { - hpb = ufshpb_get_hpb_data(sdev); - if (!hpb) - continue; - - if (hpb->is_hcm) { - /* - * For the HPB host control mode, in case device powered up and lost HPB - * information, we will set the region flag to be RGN_FLAG_UPDATE, it will - * let host reload its L2P entries(reactivate region in the UFS device). - */ - ufshpb_set_regions_update(hpb); - } else { - /* - * For the HPB device control mode, if host side receives 02h:HPB Operation - * in UPIU response, which means device recommends the host side should - * inactivate all active regions. Here we add all active regions to inactive - * list, they will be inactivated later in ufshpb_map_work_handler(). - */ - struct victim_select_info *lru_info = &hpb->lru_info; - struct ufshpb_region *rgn; - - list_for_each_entry(rgn, &lru_info->lh_lru_rgn, list_lru_rgn) - ufshpb_submit_region_inactive(hpb, rgn->rgn_idx); - - if (ufshpb_get_state(hpb) == HPB_PRESENT) - queue_work(ufshpb_wq, &hpb->map_work); - } - } -} - -/* - * This function will parse recommended active subregion information in sense - * data field of response UPIU with SAM_STAT_GOOD state. - */ -void ufshpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) -{ - struct ufshpb_lu *hpb = ufshpb_get_hpb_data(lrbp->cmd->device); - struct utp_hpb_rsp *rsp_field = &lrbp->ucd_rsp_ptr->hr; - int data_seg_len; - - data_seg_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) - & MASK_RSP_UPIU_DATA_SEG_LEN; - - /* If data segment length is zero, rsp_field is not valid */ - if (!data_seg_len) - return; - - if (unlikely(lrbp->lun != rsp_field->lun)) { - struct scsi_device *sdev; - bool found = false; - - __shost_for_each_device(sdev, hba->host) { - hpb = ufshpb_get_hpb_data(sdev); - - if (!hpb) - continue; - - if (rsp_field->lun == hpb->lun) { - found = true; - break; - } - } - - if (!found) - return; - } - - if (!hpb) - return; - - if (ufshpb_get_state(hpb) == HPB_INIT) - return; - - if ((ufshpb_get_state(hpb) != HPB_PRESENT) && - (ufshpb_get_state(hpb) != HPB_SUSPEND)) { - dev_notice(&hpb->sdev_ufs_lu->sdev_dev, - "%s: ufshpb state is not PRESENT/SUSPEND\n", - __func__); - return; - } - - BUILD_BUG_ON(sizeof(struct utp_hpb_rsp) != UTP_HPB_RSP_SIZE); - - if (!ufshpb_is_hpb_rsp_valid(hba, lrbp, rsp_field)) - return; - - hpb->stats.rcmd_noti_cnt++; - - switch (rsp_field->hpb_op) { - case HPB_RSP_REQ_REGION_UPDATE: - if (data_seg_len != DEV_DATA_SEG_LEN) - dev_warn(&hpb->sdev_ufs_lu->sdev_dev, - "%s: data seg length is not same.\n", - __func__); - ufshpb_rsp_req_region_update(hpb, rsp_field); - break; - case HPB_RSP_DEV_RESET: - dev_warn(&hpb->sdev_ufs_lu->sdev_dev, - "UFS device lost HPB information during PM.\n"); - ufshpb_dev_reset_handler(hba); - - break; - default: - dev_notice(&hpb->sdev_ufs_lu->sdev_dev, - "hpb_op is not available: %d\n", - rsp_field->hpb_op); - break; - } -} - -static void ufshpb_add_active_list(struct ufshpb_lu *hpb, - struct ufshpb_region *rgn, - struct ufshpb_subregion *srgn) -{ - if (!list_empty(&rgn->list_inact_rgn)) - return; - - if (!list_empty(&srgn->list_act_srgn)) { - list_move(&srgn->list_act_srgn, &hpb->lh_act_srgn); - return; - } - - list_add(&srgn->list_act_srgn, &hpb->lh_act_srgn); -} - -static void ufshpb_add_pending_evict_list(struct ufshpb_lu *hpb, - struct ufshpb_region *rgn, - struct list_head *pending_list) -{ - struct ufshpb_subregion *srgn; - int srgn_idx; - - if (!list_empty(&rgn->list_inact_rgn)) - return; - - for_each_sub_region(rgn, srgn_idx, srgn) - if (!list_empty(&srgn->list_act_srgn)) - return; - - list_add_tail(&rgn->list_inact_rgn, pending_list); -} - -static void ufshpb_run_active_subregion_list(struct ufshpb_lu *hpb) -{ - struct ufshpb_region *rgn; - struct ufshpb_subregion *srgn; - unsigned long flags; - int ret = 0; - - spin_lock_irqsave(&hpb->rsp_list_lock, flags); - while ((srgn = list_first_entry_or_null(&hpb->lh_act_srgn, - struct ufshpb_subregion, - list_act_srgn))) { - if (ufshpb_get_state(hpb) == HPB_SUSPEND) - break; - - list_del_init(&srgn->list_act_srgn); - spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); - - rgn = hpb->rgn_tbl + srgn->rgn_idx; - ret = ufshpb_add_region(hpb, rgn); - if (ret) - goto active_failed; - - ret = ufshpb_issue_map_req(hpb, rgn, srgn); - if (ret) { - dev_err(&hpb->sdev_ufs_lu->sdev_dev, - "issue map_req failed. ret %d, region %d - %d\n", - ret, rgn->rgn_idx, srgn->srgn_idx); - goto active_failed; - } - spin_lock_irqsave(&hpb->rsp_list_lock, flags); - } - spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); - return; - -active_failed: - dev_err(&hpb->sdev_ufs_lu->sdev_dev, "failed to activate region %d - %d, will retry\n", - rgn->rgn_idx, srgn->srgn_idx); - spin_lock_irqsave(&hpb->rsp_list_lock, flags); - ufshpb_add_active_list(hpb, rgn, srgn); - spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); -} - -static void ufshpb_run_inactive_region_list(struct ufshpb_lu *hpb) -{ - struct ufshpb_region *rgn; - unsigned long flags; - int ret; - LIST_HEAD(pending_list); - - spin_lock_irqsave(&hpb->rsp_list_lock, flags); - while ((rgn = list_first_entry_or_null(&hpb->lh_inact_rgn, - struct ufshpb_region, - list_inact_rgn))) { - if (ufshpb_get_state(hpb) == HPB_SUSPEND) - break; - - list_del_init(&rgn->list_inact_rgn); - spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); - - ret = ufshpb_evict_region(hpb, rgn); - if (ret) { - spin_lock_irqsave(&hpb->rsp_list_lock, flags); - ufshpb_add_pending_evict_list(hpb, rgn, &pending_list); - spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); - } - - spin_lock_irqsave(&hpb->rsp_list_lock, flags); - } - - list_splice(&pending_list, &hpb->lh_inact_rgn); - spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); -} - -static void ufshpb_normalization_work_handler(struct work_struct *work) -{ - struct ufshpb_lu *hpb = container_of(work, struct ufshpb_lu, - ufshpb_normalization_work); - int rgn_idx; - u8 factor = hpb->params.normalization_factor; - - for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) { - struct ufshpb_region *rgn = hpb->rgn_tbl + rgn_idx; - int srgn_idx; - - spin_lock(&rgn->rgn_lock); - rgn->reads = 0; - for (srgn_idx = 0; srgn_idx < hpb->srgns_per_rgn; srgn_idx++) { - struct ufshpb_subregion *srgn = rgn->srgn_tbl + srgn_idx; - - srgn->reads >>= factor; - rgn->reads += srgn->reads; - } - spin_unlock(&rgn->rgn_lock); - - if (rgn->rgn_state != HPB_RGN_ACTIVE || rgn->reads) - continue; - - /* if region is active but has no reads - inactivate it */ - spin_lock(&hpb->rsp_list_lock); - ufshpb_update_inactive_info(hpb, rgn->rgn_idx); - spin_unlock(&hpb->rsp_list_lock); - } -} - -static void ufshpb_map_work_handler(struct work_struct *work) -{ - struct ufshpb_lu *hpb = container_of(work, struct ufshpb_lu, map_work); - - if (ufshpb_get_state(hpb) != HPB_PRESENT) { - dev_notice(&hpb->sdev_ufs_lu->sdev_dev, - "%s: ufshpb state is not PRESENT\n", __func__); - return; - } - - ufshpb_run_inactive_region_list(hpb); - ufshpb_run_active_subregion_list(hpb); -} - -/* - * this function doesn't need to hold lock due to be called in init. - * (rgn_state_lock, rsp_list_lock, etc..) - */ -static int ufshpb_init_pinned_active_region(struct ufs_hba *hba, - struct ufshpb_lu *hpb, - struct ufshpb_region *rgn) -{ - struct ufshpb_subregion *srgn; - int srgn_idx, i; - int err = 0; - - for_each_sub_region(rgn, srgn_idx, srgn) { - srgn->mctx = ufshpb_get_map_ctx(hpb, srgn->is_last); - srgn->srgn_state = HPB_SRGN_INVALID; - if (!srgn->mctx) { - err = -ENOMEM; - dev_err(hba->dev, - "alloc mctx for pinned region failed\n"); - goto release; - } - - list_add_tail(&srgn->list_act_srgn, &hpb->lh_act_srgn); - } - - rgn->rgn_state = HPB_RGN_PINNED; - return 0; - -release: - for (i = 0; i < srgn_idx; i++) { - srgn = rgn->srgn_tbl + i; - ufshpb_put_map_ctx(hpb, srgn->mctx); - } - return err; -} - -static void ufshpb_init_subregion_tbl(struct ufshpb_lu *hpb, - struct ufshpb_region *rgn, bool last) -{ - int srgn_idx; - struct ufshpb_subregion *srgn; - - for_each_sub_region(rgn, srgn_idx, srgn) { - INIT_LIST_HEAD(&srgn->list_act_srgn); - - srgn->rgn_idx = rgn->rgn_idx; - srgn->srgn_idx = srgn_idx; - srgn->srgn_state = HPB_SRGN_UNUSED; - } - - if (unlikely(last && hpb->last_srgn_entries)) - srgn->is_last = true; -} - -static int ufshpb_alloc_subregion_tbl(struct ufshpb_lu *hpb, - struct ufshpb_region *rgn, int srgn_cnt) -{ - rgn->srgn_tbl = kvcalloc(srgn_cnt, sizeof(struct ufshpb_subregion), - GFP_KERNEL); - if (!rgn->srgn_tbl) - return -ENOMEM; - - rgn->srgn_cnt = srgn_cnt; - return 0; -} - -static void ufshpb_lu_parameter_init(struct ufs_hba *hba, - struct ufshpb_lu *hpb, - struct ufshpb_dev_info *hpb_dev_info, - struct ufshpb_lu_info *hpb_lu_info) -{ - u32 entries_per_rgn; - u64 rgn_mem_size, tmp; - - if (ufshpb_is_legacy(hba)) - hpb->pre_req_max_tr_len = HPB_LEGACY_CHUNK_HIGH; - else - hpb->pre_req_max_tr_len = hpb_dev_info->max_hpb_single_cmd; - - hpb->lu_pinned_start = hpb_lu_info->pinned_start; - hpb->lu_pinned_end = hpb_lu_info->num_pinned ? - (hpb_lu_info->pinned_start + hpb_lu_info->num_pinned - 1) - : PINNED_NOT_SET; - hpb->lru_info.max_lru_active_cnt = - hpb_lu_info->max_active_rgns - hpb_lu_info->num_pinned; - - rgn_mem_size = (1ULL << hpb_dev_info->rgn_size) * HPB_RGN_SIZE_UNIT - * HPB_ENTRY_SIZE; - do_div(rgn_mem_size, HPB_ENTRY_BLOCK_SIZE); - hpb->srgn_mem_size = (1ULL << hpb_dev_info->srgn_size) - * HPB_RGN_SIZE_UNIT / HPB_ENTRY_BLOCK_SIZE * HPB_ENTRY_SIZE; - - tmp = rgn_mem_size; - do_div(tmp, HPB_ENTRY_SIZE); - entries_per_rgn = (u32)tmp; - hpb->entries_per_rgn_shift = ilog2(entries_per_rgn); - hpb->entries_per_rgn_mask = entries_per_rgn - 1; - - hpb->entries_per_srgn = hpb->srgn_mem_size / HPB_ENTRY_SIZE; - hpb->entries_per_srgn_shift = ilog2(hpb->entries_per_srgn); - hpb->entries_per_srgn_mask = hpb->entries_per_srgn - 1; - - tmp = rgn_mem_size; - do_div(tmp, hpb->srgn_mem_size); - hpb->srgns_per_rgn = (int)tmp; - - hpb->rgns_per_lu = DIV_ROUND_UP(hpb_lu_info->num_blocks, - entries_per_rgn); - hpb->srgns_per_lu = DIV_ROUND_UP(hpb_lu_info->num_blocks, - (hpb->srgn_mem_size / HPB_ENTRY_SIZE)); - hpb->last_srgn_entries = hpb_lu_info->num_blocks - % (hpb->srgn_mem_size / HPB_ENTRY_SIZE); - - hpb->pages_per_srgn = DIV_ROUND_UP(hpb->srgn_mem_size, PAGE_SIZE); - - if (hpb_dev_info->control_mode == HPB_HOST_CONTROL) - hpb->is_hcm = true; -} - -static int ufshpb_alloc_region_tbl(struct ufs_hba *hba, struct ufshpb_lu *hpb) -{ - struct ufshpb_region *rgn_table, *rgn; - int rgn_idx, i; - int ret = 0; - - rgn_table = kvcalloc(hpb->rgns_per_lu, sizeof(struct ufshpb_region), - GFP_KERNEL); - if (!rgn_table) - return -ENOMEM; - - for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) { - int srgn_cnt = hpb->srgns_per_rgn; - bool last_srgn = false; - - rgn = rgn_table + rgn_idx; - rgn->rgn_idx = rgn_idx; - - spin_lock_init(&rgn->rgn_lock); - - INIT_LIST_HEAD(&rgn->list_inact_rgn); - INIT_LIST_HEAD(&rgn->list_lru_rgn); - INIT_LIST_HEAD(&rgn->list_expired_rgn); - - if (rgn_idx == hpb->rgns_per_lu - 1) { - srgn_cnt = ((hpb->srgns_per_lu - 1) % - hpb->srgns_per_rgn) + 1; - last_srgn = true; - } - - ret = ufshpb_alloc_subregion_tbl(hpb, rgn, srgn_cnt); - if (ret) - goto release_srgn_table; - ufshpb_init_subregion_tbl(hpb, rgn, last_srgn); - - if (ufshpb_is_pinned_region(hpb, rgn_idx)) { - ret = ufshpb_init_pinned_active_region(hba, hpb, rgn); - if (ret) - goto release_srgn_table; - } else { - rgn->rgn_state = HPB_RGN_INACTIVE; - } - - rgn->rgn_flags = 0; - rgn->hpb = hpb; - } - - hpb->rgn_tbl = rgn_table; - - return 0; - -release_srgn_table: - for (i = 0; i <= rgn_idx; i++) - kvfree(rgn_table[i].srgn_tbl); - - kvfree(rgn_table); - return ret; -} - -static void ufshpb_destroy_subregion_tbl(struct ufshpb_lu *hpb, - struct ufshpb_region *rgn) -{ - int srgn_idx; - struct ufshpb_subregion *srgn; - - for_each_sub_region(rgn, srgn_idx, srgn) - if (srgn->srgn_state != HPB_SRGN_UNUSED) { - srgn->srgn_state = HPB_SRGN_UNUSED; - ufshpb_put_map_ctx(hpb, srgn->mctx); - } -} - -static void ufshpb_destroy_region_tbl(struct ufshpb_lu *hpb) -{ - int rgn_idx; - - for (rgn_idx = 0; rgn_idx < hpb->rgns_per_lu; rgn_idx++) { - struct ufshpb_region *rgn; - - rgn = hpb->rgn_tbl + rgn_idx; - if (rgn->rgn_state != HPB_RGN_INACTIVE) { - rgn->rgn_state = HPB_RGN_INACTIVE; - - ufshpb_destroy_subregion_tbl(hpb, rgn); - } - - kvfree(rgn->srgn_tbl); - } - - kvfree(hpb->rgn_tbl); -} - -/* SYSFS functions */ -#define ufshpb_sysfs_attr_show_func(__name) \ -static ssize_t __name##_show(struct device *dev, \ - struct device_attribute *attr, char *buf) \ -{ \ - struct scsi_device *sdev = to_scsi_device(dev); \ - struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); \ - \ - if (!hpb) \ - return -ENODEV; \ - \ - return sysfs_emit(buf, "%llu\n", hpb->stats.__name); \ -} \ -\ -static DEVICE_ATTR_RO(__name) - -ufshpb_sysfs_attr_show_func(hit_cnt); -ufshpb_sysfs_attr_show_func(miss_cnt); -ufshpb_sysfs_attr_show_func(rcmd_noti_cnt); -ufshpb_sysfs_attr_show_func(rcmd_active_cnt); -ufshpb_sysfs_attr_show_func(rcmd_inactive_cnt); -ufshpb_sysfs_attr_show_func(map_req_cnt); -ufshpb_sysfs_attr_show_func(umap_req_cnt); - -static struct attribute *hpb_dev_stat_attrs[] = { - &dev_attr_hit_cnt.attr, - &dev_attr_miss_cnt.attr, - &dev_attr_rcmd_noti_cnt.attr, - &dev_attr_rcmd_active_cnt.attr, - &dev_attr_rcmd_inactive_cnt.attr, - &dev_attr_map_req_cnt.attr, - &dev_attr_umap_req_cnt.attr, - NULL, -}; - -struct attribute_group ufs_sysfs_hpb_stat_group = { - .name = "hpb_stats", - .attrs = hpb_dev_stat_attrs, -}; - -/* SYSFS functions */ -#define ufshpb_sysfs_param_show_func(__name) \ -static ssize_t __name##_show(struct device *dev, \ - struct device_attribute *attr, char *buf) \ -{ \ - struct scsi_device *sdev = to_scsi_device(dev); \ - struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); \ - \ - if (!hpb) \ - return -ENODEV; \ - \ - return sysfs_emit(buf, "%d\n", hpb->params.__name); \ -} - -ufshpb_sysfs_param_show_func(requeue_timeout_ms); -static ssize_t -requeue_timeout_ms_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct scsi_device *sdev = to_scsi_device(dev); - struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); - int val; - - if (!hpb) - return -ENODEV; - - if (kstrtouint(buf, 0, &val)) - return -EINVAL; - - if (val < 0) - return -EINVAL; - - hpb->params.requeue_timeout_ms = val; - - return count; -} -static DEVICE_ATTR_RW(requeue_timeout_ms); - -ufshpb_sysfs_param_show_func(activation_thld); -static ssize_t -activation_thld_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct scsi_device *sdev = to_scsi_device(dev); - struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); - int val; - - if (!hpb) - return -ENODEV; - - if (!hpb->is_hcm) - return -EOPNOTSUPP; - - if (kstrtouint(buf, 0, &val)) - return -EINVAL; - - if (val <= 0) - return -EINVAL; - - hpb->params.activation_thld = val; - - return count; -} -static DEVICE_ATTR_RW(activation_thld); - -ufshpb_sysfs_param_show_func(normalization_factor); -static ssize_t -normalization_factor_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct scsi_device *sdev = to_scsi_device(dev); - struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); - int val; - - if (!hpb) - return -ENODEV; - - if (!hpb->is_hcm) - return -EOPNOTSUPP; - - if (kstrtouint(buf, 0, &val)) - return -EINVAL; - - if (val <= 0 || val > ilog2(hpb->entries_per_srgn)) - return -EINVAL; - - hpb->params.normalization_factor = val; - - return count; -} -static DEVICE_ATTR_RW(normalization_factor); - -ufshpb_sysfs_param_show_func(eviction_thld_enter); -static ssize_t -eviction_thld_enter_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct scsi_device *sdev = to_scsi_device(dev); - struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); - int val; - - if (!hpb) - return -ENODEV; - - if (!hpb->is_hcm) - return -EOPNOTSUPP; - - if (kstrtouint(buf, 0, &val)) - return -EINVAL; - - if (val <= hpb->params.eviction_thld_exit) - return -EINVAL; - - hpb->params.eviction_thld_enter = val; - - return count; -} -static DEVICE_ATTR_RW(eviction_thld_enter); - -ufshpb_sysfs_param_show_func(eviction_thld_exit); -static ssize_t -eviction_thld_exit_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct scsi_device *sdev = to_scsi_device(dev); - struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); - int val; - - if (!hpb) - return -ENODEV; - - if (!hpb->is_hcm) - return -EOPNOTSUPP; - - if (kstrtouint(buf, 0, &val)) - return -EINVAL; - - if (val <= hpb->params.activation_thld) - return -EINVAL; - - hpb->params.eviction_thld_exit = val; - - return count; -} -static DEVICE_ATTR_RW(eviction_thld_exit); - -ufshpb_sysfs_param_show_func(read_timeout_ms); -static ssize_t -read_timeout_ms_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct scsi_device *sdev = to_scsi_device(dev); - struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); - int val; - - if (!hpb) - return -ENODEV; - - if (!hpb->is_hcm) - return -EOPNOTSUPP; - - if (kstrtouint(buf, 0, &val)) - return -EINVAL; - - /* read_timeout >> timeout_polling_interval */ - if (val < hpb->params.timeout_polling_interval_ms * 2) - return -EINVAL; - - hpb->params.read_timeout_ms = val; - - return count; -} -static DEVICE_ATTR_RW(read_timeout_ms); - -ufshpb_sysfs_param_show_func(read_timeout_expiries); -static ssize_t -read_timeout_expiries_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct scsi_device *sdev = to_scsi_device(dev); - struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); - int val; - - if (!hpb) - return -ENODEV; - - if (!hpb->is_hcm) - return -EOPNOTSUPP; - - if (kstrtouint(buf, 0, &val)) - return -EINVAL; - - if (val <= 0) - return -EINVAL; - - hpb->params.read_timeout_expiries = val; - - return count; -} -static DEVICE_ATTR_RW(read_timeout_expiries); - -ufshpb_sysfs_param_show_func(timeout_polling_interval_ms); -static ssize_t -timeout_polling_interval_ms_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct scsi_device *sdev = to_scsi_device(dev); - struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); - int val; - - if (!hpb) - return -ENODEV; - - if (!hpb->is_hcm) - return -EOPNOTSUPP; - - if (kstrtouint(buf, 0, &val)) - return -EINVAL; - - /* timeout_polling_interval << read_timeout */ - if (val <= 0 || val > hpb->params.read_timeout_ms / 2) - return -EINVAL; - - hpb->params.timeout_polling_interval_ms = val; - - return count; -} -static DEVICE_ATTR_RW(timeout_polling_interval_ms); - -ufshpb_sysfs_param_show_func(inflight_map_req); -static ssize_t inflight_map_req_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct scsi_device *sdev = to_scsi_device(dev); - struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); - int val; - - if (!hpb) - return -ENODEV; - - if (!hpb->is_hcm) - return -EOPNOTSUPP; - - if (kstrtouint(buf, 0, &val)) - return -EINVAL; - - if (val <= 0 || val > hpb->sdev_ufs_lu->queue_depth - 1) - return -EINVAL; - - hpb->params.inflight_map_req = val; - - return count; -} -static DEVICE_ATTR_RW(inflight_map_req); - -static void ufshpb_hcm_param_init(struct ufshpb_lu *hpb) -{ - hpb->params.activation_thld = ACTIVATION_THRESHOLD; - hpb->params.normalization_factor = 1; - hpb->params.eviction_thld_enter = (ACTIVATION_THRESHOLD << 5); - hpb->params.eviction_thld_exit = (ACTIVATION_THRESHOLD << 4); - hpb->params.read_timeout_ms = READ_TO_MS; - hpb->params.read_timeout_expiries = READ_TO_EXPIRIES; - hpb->params.timeout_polling_interval_ms = POLLING_INTERVAL_MS; - hpb->params.inflight_map_req = THROTTLE_MAP_REQ_DEFAULT; -} - -static struct attribute *hpb_dev_param_attrs[] = { - &dev_attr_requeue_timeout_ms.attr, - &dev_attr_activation_thld.attr, - &dev_attr_normalization_factor.attr, - &dev_attr_eviction_thld_enter.attr, - &dev_attr_eviction_thld_exit.attr, - &dev_attr_read_timeout_ms.attr, - &dev_attr_read_timeout_expiries.attr, - &dev_attr_timeout_polling_interval_ms.attr, - &dev_attr_inflight_map_req.attr, - NULL, -}; - -struct attribute_group ufs_sysfs_hpb_param_group = { - .name = "hpb_params", - .attrs = hpb_dev_param_attrs, -}; - -static int ufshpb_pre_req_mempool_init(struct ufshpb_lu *hpb) -{ - struct ufshpb_req *pre_req = NULL, *t; - int qd = hpb->sdev_ufs_lu->queue_depth / 2; - int i; - - INIT_LIST_HEAD(&hpb->lh_pre_req_free); - - hpb->pre_req = kcalloc(qd, sizeof(struct ufshpb_req), GFP_KERNEL); - hpb->throttle_pre_req = qd; - hpb->num_inflight_pre_req = 0; - - if (!hpb->pre_req) - goto release_mem; - - for (i = 0; i < qd; i++) { - pre_req = hpb->pre_req + i; - INIT_LIST_HEAD(&pre_req->list_req); - pre_req->req = NULL; - - pre_req->bio = bio_alloc(NULL, 1, 0, GFP_KERNEL); - if (!pre_req->bio) - goto release_mem; - - pre_req->wb.m_page = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (!pre_req->wb.m_page) { - bio_put(pre_req->bio); - goto release_mem; - } - - list_add_tail(&pre_req->list_req, &hpb->lh_pre_req_free); - } - - return 0; -release_mem: - list_for_each_entry_safe(pre_req, t, &hpb->lh_pre_req_free, list_req) { - list_del_init(&pre_req->list_req); - bio_put(pre_req->bio); - __free_page(pre_req->wb.m_page); - } - - kfree(hpb->pre_req); - return -ENOMEM; -} - -static void ufshpb_pre_req_mempool_destroy(struct ufshpb_lu *hpb) -{ - struct ufshpb_req *pre_req = NULL; - int i; - - for (i = 0; i < hpb->throttle_pre_req; i++) { - pre_req = hpb->pre_req + i; - bio_put(hpb->pre_req[i].bio); - if (!pre_req->wb.m_page) - __free_page(hpb->pre_req[i].wb.m_page); - list_del_init(&pre_req->list_req); - } - - kfree(hpb->pre_req); -} - -static void ufshpb_stat_init(struct ufshpb_lu *hpb) -{ - hpb->stats.hit_cnt = 0; - hpb->stats.miss_cnt = 0; - hpb->stats.rcmd_noti_cnt = 0; - hpb->stats.rcmd_active_cnt = 0; - hpb->stats.rcmd_inactive_cnt = 0; - hpb->stats.map_req_cnt = 0; - hpb->stats.umap_req_cnt = 0; -} - -static void ufshpb_param_init(struct ufshpb_lu *hpb) -{ - hpb->params.requeue_timeout_ms = HPB_REQUEUE_TIME_MS; - if (hpb->is_hcm) - ufshpb_hcm_param_init(hpb); -} - -static int ufshpb_lu_hpb_init(struct ufs_hba *hba, struct ufshpb_lu *hpb) -{ - int ret; - - spin_lock_init(&hpb->rgn_state_lock); - spin_lock_init(&hpb->rsp_list_lock); - spin_lock_init(&hpb->param_lock); - - INIT_LIST_HEAD(&hpb->lru_info.lh_lru_rgn); - INIT_LIST_HEAD(&hpb->lh_act_srgn); - INIT_LIST_HEAD(&hpb->lh_inact_rgn); - INIT_LIST_HEAD(&hpb->list_hpb_lu); - - INIT_WORK(&hpb->map_work, ufshpb_map_work_handler); - if (hpb->is_hcm) { - INIT_WORK(&hpb->ufshpb_normalization_work, - ufshpb_normalization_work_handler); - INIT_DELAYED_WORK(&hpb->ufshpb_read_to_work, - ufshpb_read_to_handler); - } - - hpb->map_req_cache = kmem_cache_create("ufshpb_req_cache", - sizeof(struct ufshpb_req), 0, 0, NULL); - if (!hpb->map_req_cache) { - dev_err(hba->dev, "ufshpb(%d) ufshpb_req_cache create fail", - hpb->lun); - return -ENOMEM; - } - - hpb->m_page_cache = kmem_cache_create("ufshpb_m_page_cache", - sizeof(struct page *) * hpb->pages_per_srgn, - 0, 0, NULL); - if (!hpb->m_page_cache) { - dev_err(hba->dev, "ufshpb(%d) ufshpb_m_page_cache create fail", - hpb->lun); - ret = -ENOMEM; - goto release_req_cache; - } - - ret = ufshpb_pre_req_mempool_init(hpb); - if (ret) { - dev_err(hba->dev, "ufshpb(%d) pre_req_mempool init fail", - hpb->lun); - goto release_m_page_cache; - } - - ret = ufshpb_alloc_region_tbl(hba, hpb); - if (ret) - goto release_pre_req_mempool; - - ufshpb_stat_init(hpb); - ufshpb_param_init(hpb); - - if (hpb->is_hcm) { - unsigned int poll; - - poll = hpb->params.timeout_polling_interval_ms; - schedule_delayed_work(&hpb->ufshpb_read_to_work, - msecs_to_jiffies(poll)); - } - - return 0; - -release_pre_req_mempool: - ufshpb_pre_req_mempool_destroy(hpb); -release_m_page_cache: - kmem_cache_destroy(hpb->m_page_cache); -release_req_cache: - kmem_cache_destroy(hpb->map_req_cache); - return ret; -} - -static struct ufshpb_lu * -ufshpb_alloc_hpb_lu(struct ufs_hba *hba, struct scsi_device *sdev, - struct ufshpb_dev_info *hpb_dev_info, - struct ufshpb_lu_info *hpb_lu_info) -{ - struct ufshpb_lu *hpb; - int ret; - - hpb = kzalloc(sizeof(struct ufshpb_lu), GFP_KERNEL); - if (!hpb) - return NULL; - - hpb->lun = sdev->lun; - hpb->sdev_ufs_lu = sdev; - - ufshpb_lu_parameter_init(hba, hpb, hpb_dev_info, hpb_lu_info); - - ret = ufshpb_lu_hpb_init(hba, hpb); - if (ret) { - dev_err(hba->dev, "hpb lu init failed. ret %d", ret); - goto release_hpb; - } - - sdev->hostdata = hpb; - return hpb; - -release_hpb: - kfree(hpb); - return NULL; -} - -static void ufshpb_discard_rsp_lists(struct ufshpb_lu *hpb) -{ - struct ufshpb_region *rgn, *next_rgn; - struct ufshpb_subregion *srgn, *next_srgn; - unsigned long flags; - - /* - * If the device reset occurred, the remaining HPB region information - * may be stale. Therefore, by discarding the lists of HPB response - * that remained after reset, we prevent unnecessary work. - */ - spin_lock_irqsave(&hpb->rsp_list_lock, flags); - list_for_each_entry_safe(rgn, next_rgn, &hpb->lh_inact_rgn, - list_inact_rgn) - list_del_init(&rgn->list_inact_rgn); - - list_for_each_entry_safe(srgn, next_srgn, &hpb->lh_act_srgn, - list_act_srgn) - list_del_init(&srgn->list_act_srgn); - spin_unlock_irqrestore(&hpb->rsp_list_lock, flags); -} - -static void ufshpb_cancel_jobs(struct ufshpb_lu *hpb) -{ - if (hpb->is_hcm) { - cancel_delayed_work_sync(&hpb->ufshpb_read_to_work); - cancel_work_sync(&hpb->ufshpb_normalization_work); - } - cancel_work_sync(&hpb->map_work); -} - -static bool ufshpb_check_hpb_reset_query(struct ufs_hba *hba) -{ - int err = 0; - bool flag_res = true; - int try; - - /* wait for the device to complete HPB reset query */ - for (try = 0; try < HPB_RESET_REQ_RETRIES; try++) { - dev_dbg(hba->dev, - "%s: start flag reset polling %d times\n", - __func__, try); - - /* Poll fHpbReset flag to be cleared */ - err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG, - QUERY_FLAG_IDN_HPB_RESET, 0, &flag_res); - - if (err) { - dev_err(hba->dev, - "%s: reading fHpbReset flag failed with error %d\n", - __func__, err); - return flag_res; - } - - if (!flag_res) - goto out; - - usleep_range(1000, 1100); - } - if (flag_res) { - dev_err(hba->dev, - "%s: fHpbReset was not cleared by the device\n", - __func__); - } -out: - return flag_res; -} - -/** - * ufshpb_toggle_state - switch HPB state of all LUs - * @hba: per-adapter instance - * @src: expected current HPB state - * @dest: target HPB state to switch to - */ -void ufshpb_toggle_state(struct ufs_hba *hba, enum UFSHPB_STATE src, enum UFSHPB_STATE dest) -{ - struct ufshpb_lu *hpb; - struct scsi_device *sdev; - - shost_for_each_device(sdev, hba->host) { - hpb = ufshpb_get_hpb_data(sdev); - - if (!hpb || ufshpb_get_state(hpb) != src) - continue; - ufshpb_set_state(hpb, dest); - - if (dest == HPB_RESET) { - ufshpb_cancel_jobs(hpb); - ufshpb_discard_rsp_lists(hpb); - } - } -} - -void ufshpb_suspend(struct ufs_hba *hba) -{ - struct ufshpb_lu *hpb; - struct scsi_device *sdev; - - shost_for_each_device(sdev, hba->host) { - hpb = ufshpb_get_hpb_data(sdev); - if (!hpb || ufshpb_get_state(hpb) != HPB_PRESENT) - continue; - - ufshpb_set_state(hpb, HPB_SUSPEND); - ufshpb_cancel_jobs(hpb); - } -} - -void ufshpb_resume(struct ufs_hba *hba) -{ - struct ufshpb_lu *hpb; - struct scsi_device *sdev; - - shost_for_each_device(sdev, hba->host) { - hpb = ufshpb_get_hpb_data(sdev); - if (!hpb || ufshpb_get_state(hpb) != HPB_SUSPEND) - continue; - - ufshpb_set_state(hpb, HPB_PRESENT); - ufshpb_kick_map_work(hpb); - if (hpb->is_hcm) { - unsigned int poll = hpb->params.timeout_polling_interval_ms; - - schedule_delayed_work(&hpb->ufshpb_read_to_work, msecs_to_jiffies(poll)); - } - } -} - -static int ufshpb_get_lu_info(struct ufs_hba *hba, int lun, - struct ufshpb_lu_info *hpb_lu_info) -{ - u16 max_active_rgns; - u8 lu_enable; - int size = QUERY_DESC_MAX_SIZE; - int ret; - char desc_buf[QUERY_DESC_MAX_SIZE]; - - ufshcd_rpm_get_sync(hba); - ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, - QUERY_DESC_IDN_UNIT, lun, 0, - desc_buf, &size); - ufshcd_rpm_put_sync(hba); - - if (ret) { - dev_err(hba->dev, - "%s: idn: %d lun: %d query request failed", - __func__, QUERY_DESC_IDN_UNIT, lun); - return ret; - } - - lu_enable = desc_buf[UNIT_DESC_PARAM_LU_ENABLE]; - if (lu_enable != LU_ENABLED_HPB_FUNC) - return -ENODEV; - - max_active_rgns = get_unaligned_be16( - desc_buf + UNIT_DESC_PARAM_HPB_LU_MAX_ACTIVE_RGNS); - if (!max_active_rgns) { - dev_err(hba->dev, - "lun %d wrong number of max active regions\n", lun); - return -ENODEV; - } - - hpb_lu_info->num_blocks = get_unaligned_be64( - desc_buf + UNIT_DESC_PARAM_LOGICAL_BLK_COUNT); - hpb_lu_info->pinned_start = get_unaligned_be16( - desc_buf + UNIT_DESC_PARAM_HPB_PIN_RGN_START_OFF); - hpb_lu_info->num_pinned = get_unaligned_be16( - desc_buf + UNIT_DESC_PARAM_HPB_NUM_PIN_RGNS); - hpb_lu_info->max_active_rgns = max_active_rgns; - - return 0; -} - -void ufshpb_destroy_lu(struct ufs_hba *hba, struct scsi_device *sdev) -{ - struct ufshpb_lu *hpb = ufshpb_get_hpb_data(sdev); - - if (!hpb) - return; - - ufshpb_set_state(hpb, HPB_FAILED); - - sdev = hpb->sdev_ufs_lu; - sdev->hostdata = NULL; - - ufshpb_cancel_jobs(hpb); - - ufshpb_pre_req_mempool_destroy(hpb); - ufshpb_destroy_region_tbl(hpb); - - kmem_cache_destroy(hpb->map_req_cache); - kmem_cache_destroy(hpb->m_page_cache); - - list_del_init(&hpb->list_hpb_lu); - - kfree(hpb); -} - -static void ufshpb_hpb_lu_prepared(struct ufs_hba *hba) -{ - int pool_size; - struct ufshpb_lu *hpb; - struct scsi_device *sdev; - bool init_success; - - if (tot_active_srgn_pages == 0) { - ufshpb_remove(hba); - return; - } - - init_success = !ufshpb_check_hpb_reset_query(hba); - - pool_size = PAGE_ALIGN(ufshpb_host_map_kbytes * SZ_1K) / PAGE_SIZE; - if (pool_size > tot_active_srgn_pages) { - mempool_resize(ufshpb_mctx_pool, tot_active_srgn_pages); - mempool_resize(ufshpb_page_pool, tot_active_srgn_pages); - } - - shost_for_each_device(sdev, hba->host) { - hpb = ufshpb_get_hpb_data(sdev); - if (!hpb) - continue; - - if (init_success) { - ufshpb_set_state(hpb, HPB_PRESENT); - if ((hpb->lu_pinned_end - hpb->lu_pinned_start) > 0) - queue_work(ufshpb_wq, &hpb->map_work); - } else { - dev_err(hba->dev, "destroy HPB lu %d\n", hpb->lun); - ufshpb_destroy_lu(hba, sdev); - } - } - - if (!init_success) - ufshpb_remove(hba); -} - -void ufshpb_init_hpb_lu(struct ufs_hba *hba, struct scsi_device *sdev) -{ - struct ufshpb_lu *hpb; - int ret; - struct ufshpb_lu_info hpb_lu_info = { 0 }; - int lun = sdev->lun; - - if (lun >= hba->dev_info.max_lu_supported) - goto out; - - ret = ufshpb_get_lu_info(hba, lun, &hpb_lu_info); - if (ret) - goto out; - - hpb = ufshpb_alloc_hpb_lu(hba, sdev, &hba->ufshpb_dev, - &hpb_lu_info); - if (!hpb) - goto out; - - tot_active_srgn_pages += hpb_lu_info.max_active_rgns * - hpb->srgns_per_rgn * hpb->pages_per_srgn; - -out: - /* All LUs are initialized */ - if (atomic_dec_and_test(&hba->ufshpb_dev.slave_conf_cnt)) - ufshpb_hpb_lu_prepared(hba); -} - -static int ufshpb_init_mem_wq(struct ufs_hba *hba) -{ - int ret; - unsigned int pool_size; - - ufshpb_mctx_cache = kmem_cache_create("ufshpb_mctx_cache", - sizeof(struct ufshpb_map_ctx), - 0, 0, NULL); - if (!ufshpb_mctx_cache) { - dev_err(hba->dev, "ufshpb: cannot init mctx cache\n"); - return -ENOMEM; - } - - pool_size = PAGE_ALIGN(ufshpb_host_map_kbytes * SZ_1K) / PAGE_SIZE; - dev_info(hba->dev, "%s:%d ufshpb_host_map_kbytes %u pool_size %u\n", - __func__, __LINE__, ufshpb_host_map_kbytes, pool_size); - - ufshpb_mctx_pool = mempool_create_slab_pool(pool_size, - ufshpb_mctx_cache); - if (!ufshpb_mctx_pool) { - dev_err(hba->dev, "ufshpb: cannot init mctx pool\n"); - ret = -ENOMEM; - goto release_mctx_cache; - } - - ufshpb_page_pool = mempool_create_page_pool(pool_size, 0); - if (!ufshpb_page_pool) { - dev_err(hba->dev, "ufshpb: cannot init page pool\n"); - ret = -ENOMEM; - goto release_mctx_pool; - } - - ufshpb_wq = alloc_workqueue("ufshpb-wq", - WQ_UNBOUND | WQ_MEM_RECLAIM, 0); - if (!ufshpb_wq) { - dev_err(hba->dev, "ufshpb: alloc workqueue failed\n"); - ret = -ENOMEM; - goto release_page_pool; - } - - return 0; - -release_page_pool: - mempool_destroy(ufshpb_page_pool); -release_mctx_pool: - mempool_destroy(ufshpb_mctx_pool); -release_mctx_cache: - kmem_cache_destroy(ufshpb_mctx_cache); - return ret; -} - -void ufshpb_get_geo_info(struct ufs_hba *hba, u8 *geo_buf) -{ - struct ufshpb_dev_info *hpb_info = &hba->ufshpb_dev; - int max_active_rgns = 0; - int hpb_num_lu; - - hpb_num_lu = geo_buf[GEOMETRY_DESC_PARAM_HPB_NUMBER_LU]; - if (hpb_num_lu == 0) { - dev_err(hba->dev, "No HPB LU supported\n"); - hpb_info->hpb_disabled = true; - return; - } - - hpb_info->rgn_size = geo_buf[GEOMETRY_DESC_PARAM_HPB_REGION_SIZE]; - hpb_info->srgn_size = geo_buf[GEOMETRY_DESC_PARAM_HPB_SUBREGION_SIZE]; - max_active_rgns = get_unaligned_be16(geo_buf + - GEOMETRY_DESC_PARAM_HPB_MAX_ACTIVE_REGS); - - if (hpb_info->rgn_size == 0 || hpb_info->srgn_size == 0 || - max_active_rgns == 0) { - dev_err(hba->dev, "No HPB supported device\n"); - hpb_info->hpb_disabled = true; - return; - } -} - -void ufshpb_get_dev_info(struct ufs_hba *hba, u8 *desc_buf) -{ - struct ufshpb_dev_info *hpb_dev_info = &hba->ufshpb_dev; - int version, ret; - int max_single_cmd; - - hpb_dev_info->control_mode = desc_buf[DEVICE_DESC_PARAM_HPB_CONTROL]; - - version = get_unaligned_be16(desc_buf + DEVICE_DESC_PARAM_HPB_VER); - if ((version != HPB_SUPPORT_VERSION) && - (version != HPB_SUPPORT_LEGACY_VERSION)) { - dev_err(hba->dev, "%s: HPB %x version is not supported.\n", - __func__, version); - hpb_dev_info->hpb_disabled = true; - return; - } - - if (version == HPB_SUPPORT_LEGACY_VERSION) - hpb_dev_info->is_legacy = true; - - /* - * Get the number of user logical unit to check whether all - * scsi_device finish initialization - */ - hpb_dev_info->num_lu = desc_buf[DEVICE_DESC_PARAM_NUM_LU]; - - if (hpb_dev_info->is_legacy) - return; - - ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, - QUERY_ATTR_IDN_MAX_HPB_SINGLE_CMD, 0, 0, &max_single_cmd); - - if (ret) - hpb_dev_info->max_hpb_single_cmd = HPB_LEGACY_CHUNK_HIGH; - else - hpb_dev_info->max_hpb_single_cmd = min(max_single_cmd + 1, HPB_MULTI_CHUNK_HIGH); -} - -void ufshpb_init(struct ufs_hba *hba) -{ - struct ufshpb_dev_info *hpb_dev_info = &hba->ufshpb_dev; - int try; - int ret; - - if (!ufshpb_is_allowed(hba) || !hba->dev_info.hpb_enabled) - return; - - if (ufshpb_init_mem_wq(hba)) { - hpb_dev_info->hpb_disabled = true; - return; - } - - atomic_set(&hpb_dev_info->slave_conf_cnt, hpb_dev_info->num_lu); - tot_active_srgn_pages = 0; - /* issue HPB reset query */ - for (try = 0; try < HPB_RESET_REQ_RETRIES; try++) { - ret = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_SET_FLAG, - QUERY_FLAG_IDN_HPB_RESET, 0, NULL); - if (!ret) - break; - } -} - -void ufshpb_remove(struct ufs_hba *hba) -{ - mempool_destroy(ufshpb_page_pool); - mempool_destroy(ufshpb_mctx_pool); - kmem_cache_destroy(ufshpb_mctx_cache); - - destroy_workqueue(ufshpb_wq); -} - -module_param(ufshpb_host_map_kbytes, uint, 0644); -MODULE_PARM_DESC(ufshpb_host_map_kbytes, - "ufshpb host mapping memory kilo-bytes for ufshpb memory-pool"); diff --git a/drivers/ufs/core/ufshpb.h b/drivers/ufs/core/ufshpb.h deleted file mode 100644 index b428bbdd2799..000000000000 --- a/drivers/ufs/core/ufshpb.h +++ /dev/null @@ -1,318 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Universal Flash Storage Host Performance Booster - * - * Copyright (C) 2017-2021 Samsung Electronics Co., Ltd. - * - * Authors: - * Yongmyung Lee <ymhungry.lee@samsung.com> - * Jinyoung Choi <j-young.choi@samsung.com> - */ - -#ifndef _UFSHPB_H_ -#define _UFSHPB_H_ - -/* hpb response UPIU macro */ -#define HPB_RSP_NONE 0x0 -#define HPB_RSP_REQ_REGION_UPDATE 0x1 -#define HPB_RSP_DEV_RESET 0x2 -#define MAX_ACTIVE_NUM 2 -#define MAX_INACTIVE_NUM 2 -#define DEV_DATA_SEG_LEN 0x14 -#define DEV_SENSE_SEG_LEN 0x12 -#define DEV_DES_TYPE 0x80 -#define DEV_ADDITIONAL_LEN 0x10 - -/* hpb map & entries macro */ -#define HPB_RGN_SIZE_UNIT 512 -#define HPB_ENTRY_BLOCK_SIZE SZ_4K -#define HPB_ENTRY_SIZE 0x8 -#define PINNED_NOT_SET U32_MAX - -/* hpb support chunk size */ -#define HPB_LEGACY_CHUNK_HIGH 1 -#define HPB_MULTI_CHUNK_HIGH 255 - -/* hpb vender defined opcode */ -#define UFSHPB_READ 0xF8 -#define UFSHPB_READ_BUFFER 0xF9 -#define UFSHPB_READ_BUFFER_ID 0x01 -#define UFSHPB_WRITE_BUFFER 0xFA -#define UFSHPB_WRITE_BUFFER_INACT_SINGLE_ID 0x01 -#define UFSHPB_WRITE_BUFFER_PREFETCH_ID 0x02 -#define UFSHPB_WRITE_BUFFER_INACT_ALL_ID 0x03 -#define HPB_WRITE_BUFFER_CMD_LENGTH 10 -#define MAX_HPB_READ_ID 0x7F -#define HPB_READ_BUFFER_CMD_LENGTH 10 -#define LU_ENABLED_HPB_FUNC 0x02 - -#define HPB_RESET_REQ_RETRIES 10 -#define HPB_MAP_REQ_RETRIES 5 -#define HPB_REQUEUE_TIME_MS 0 - -#define HPB_SUPPORT_VERSION 0x200 -#define HPB_SUPPORT_LEGACY_VERSION 0x100 - -enum UFSHPB_MODE { - HPB_HOST_CONTROL, - HPB_DEVICE_CONTROL, -}; - -enum UFSHPB_STATE { - HPB_INIT, - HPB_PRESENT, - HPB_SUSPEND, - HPB_FAILED, - HPB_RESET, -}; - -enum HPB_RGN_STATE { - HPB_RGN_INACTIVE, - HPB_RGN_ACTIVE, - /* pinned regions are always active */ - HPB_RGN_PINNED, -}; - -enum HPB_SRGN_STATE { - HPB_SRGN_UNUSED, - HPB_SRGN_INVALID, - HPB_SRGN_VALID, - HPB_SRGN_ISSUED, -}; - -/** - * struct ufshpb_lu_info - UFSHPB logical unit related info - * @num_blocks: the number of logical block - * @pinned_start: the start region number of pinned region - * @num_pinned: the number of pinned regions - * @max_active_rgns: maximum number of active regions - */ -struct ufshpb_lu_info { - int num_blocks; - int pinned_start; - int num_pinned; - int max_active_rgns; -}; - -struct ufshpb_map_ctx { - struct page **m_page; - unsigned long *ppn_dirty; -}; - -struct ufshpb_subregion { - struct ufshpb_map_ctx *mctx; - enum HPB_SRGN_STATE srgn_state; - int rgn_idx; - int srgn_idx; - bool is_last; - - /* subregion reads - for host mode */ - unsigned int reads; - - /* below information is used by rsp_list */ - struct list_head list_act_srgn; -}; - -struct ufshpb_region { - struct ufshpb_lu *hpb; - struct ufshpb_subregion *srgn_tbl; - enum HPB_RGN_STATE rgn_state; - int rgn_idx; - int srgn_cnt; - - /* below information is used by rsp_list */ - struct list_head list_inact_rgn; - - /* below information is used by lru */ - struct list_head list_lru_rgn; - unsigned long rgn_flags; -#define RGN_FLAG_DIRTY 0 -#define RGN_FLAG_UPDATE 1 - - /* region reads - for host mode */ - spinlock_t rgn_lock; - unsigned int reads; - /* region "cold" timer - for host mode */ - ktime_t read_timeout; - unsigned int read_timeout_expiries; - struct list_head list_expired_rgn; -}; - -#define for_each_sub_region(rgn, i, srgn) \ - for ((i) = 0; \ - ((i) < (rgn)->srgn_cnt) && ((srgn) = &(rgn)->srgn_tbl[i]); \ - (i)++) - -/** - * struct ufshpb_req - HPB related request structure (write/read buffer) - * @req: block layer request structure - * @bio: bio for this request - * @hpb: ufshpb_lu structure that related to - * @list_req: ufshpb_req mempool list - * @sense: store its sense data - * @mctx: L2P map information - * @rgn_idx: target region index - * @srgn_idx: target sub-region index - * @lun: target logical unit number - * @m_page: L2P map information data for pre-request - * @len: length of host-side cached L2P map in m_page - * @lpn: start LPN of L2P map in m_page - */ -struct ufshpb_req { - struct request *req; - struct bio *bio; - struct ufshpb_lu *hpb; - struct list_head list_req; - union { - struct { - struct ufshpb_map_ctx *mctx; - unsigned int rgn_idx; - unsigned int srgn_idx; - unsigned int lun; - } rb; - struct { - struct page *m_page; - unsigned int len; - unsigned long lpn; - } wb; - }; -}; - -struct victim_select_info { - struct list_head lh_lru_rgn; /* LRU list of regions */ - int max_lru_active_cnt; /* supported hpb #region - pinned #region */ - atomic_t active_cnt; -}; - -/** - * ufshpb_params - ufs hpb parameters - * @requeue_timeout_ms - requeue threshold of wb command (0x2) - * @activation_thld - min reads [IOs] to activate/update a region - * @normalization_factor - shift right the region's reads - * @eviction_thld_enter - min reads [IOs] for the entering region in eviction - * @eviction_thld_exit - max reads [IOs] for the exiting region in eviction - * @read_timeout_ms - timeout [ms] from the last read IO to the region - * @read_timeout_expiries - amount of allowable timeout expireis - * @timeout_polling_interval_ms - frequency in which timeouts are checked - * @inflight_map_req - number of inflight map requests - */ -struct ufshpb_params { - unsigned int requeue_timeout_ms; - unsigned int activation_thld; - unsigned int normalization_factor; - unsigned int eviction_thld_enter; - unsigned int eviction_thld_exit; - unsigned int read_timeout_ms; - unsigned int read_timeout_expiries; - unsigned int timeout_polling_interval_ms; - unsigned int inflight_map_req; -}; - -struct ufshpb_stats { - u64 hit_cnt; - u64 miss_cnt; - u64 rcmd_noti_cnt; - u64 rcmd_active_cnt; - u64 rcmd_inactive_cnt; - u64 map_req_cnt; - u64 pre_req_cnt; - u64 umap_req_cnt; -}; - -struct ufshpb_lu { - int lun; - struct scsi_device *sdev_ufs_lu; - - spinlock_t rgn_state_lock; /* for protect rgn/srgn state */ - struct ufshpb_region *rgn_tbl; - - atomic_t hpb_state; - - spinlock_t rsp_list_lock; - struct list_head lh_act_srgn; /* hold rsp_list_lock */ - struct list_head lh_inact_rgn; /* hold rsp_list_lock */ - - /* pre request information */ - struct ufshpb_req *pre_req; - int num_inflight_pre_req; - int throttle_pre_req; - int num_inflight_map_req; /* hold param_lock */ - spinlock_t param_lock; - - struct list_head lh_pre_req_free; - int pre_req_max_tr_len; - - /* cached L2P map management worker */ - struct work_struct map_work; - - /* for selecting victim */ - struct victim_select_info lru_info; - struct work_struct ufshpb_normalization_work; - struct delayed_work ufshpb_read_to_work; - unsigned long work_data_bits; -#define TIMEOUT_WORK_RUNNING 0 - - /* pinned region information */ - u32 lu_pinned_start; - u32 lu_pinned_end; - - /* HPB related configuration */ - u32 rgns_per_lu; - u32 srgns_per_lu; - u32 last_srgn_entries; - int srgns_per_rgn; - u32 srgn_mem_size; - u32 entries_per_rgn_mask; - u32 entries_per_rgn_shift; - u32 entries_per_srgn; - u32 entries_per_srgn_mask; - u32 entries_per_srgn_shift; - u32 pages_per_srgn; - - bool is_hcm; - - struct ufshpb_stats stats; - struct ufshpb_params params; - - struct kmem_cache *map_req_cache; - struct kmem_cache *m_page_cache; - - struct list_head list_hpb_lu; -}; - -struct ufs_hba; -struct ufshcd_lrb; - -#ifndef CONFIG_SCSI_UFS_HPB -static int ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { return 0; } -static void ufshpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) {} -static void ufshpb_resume(struct ufs_hba *hba) {} -static void ufshpb_suspend(struct ufs_hba *hba) {} -static void ufshpb_toggle_state(struct ufs_hba *hba, enum UFSHPB_STATE src, enum UFSHPB_STATE dest) {} -static void ufshpb_init(struct ufs_hba *hba) {} -static void ufshpb_init_hpb_lu(struct ufs_hba *hba, struct scsi_device *sdev) {} -static void ufshpb_destroy_lu(struct ufs_hba *hba, struct scsi_device *sdev) {} -static void ufshpb_remove(struct ufs_hba *hba) {} -static bool ufshpb_is_allowed(struct ufs_hba *hba) { return false; } -static void ufshpb_get_geo_info(struct ufs_hba *hba, u8 *geo_buf) {} -static void ufshpb_get_dev_info(struct ufs_hba *hba, u8 *desc_buf) {} -static bool ufshpb_is_legacy(struct ufs_hba *hba) { return false; } -#else -int ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp); -void ufshpb_rsp_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp); -void ufshpb_resume(struct ufs_hba *hba); -void ufshpb_suspend(struct ufs_hba *hba); -void ufshpb_toggle_state(struct ufs_hba *hba, enum UFSHPB_STATE src, enum UFSHPB_STATE dest); -void ufshpb_init(struct ufs_hba *hba); -void ufshpb_init_hpb_lu(struct ufs_hba *hba, struct scsi_device *sdev); -void ufshpb_destroy_lu(struct ufs_hba *hba, struct scsi_device *sdev); -void ufshpb_remove(struct ufs_hba *hba); -bool ufshpb_is_allowed(struct ufs_hba *hba); -void ufshpb_get_geo_info(struct ufs_hba *hba, u8 *geo_buf); -void ufshpb_get_dev_info(struct ufs_hba *hba, u8 *desc_buf); -bool ufshpb_is_legacy(struct ufs_hba *hba); -extern struct attribute_group ufs_sysfs_hpb_stat_group; -extern struct attribute_group ufs_sysfs_hpb_param_group; -#endif - -#endif /* End of Header */ diff --git a/drivers/ufs/host/cdns-pltfrm.c b/drivers/ufs/host/cdns-pltfrm.c index 26761425a76c..2491e7e87028 100644 --- a/drivers/ufs/host/cdns-pltfrm.c +++ b/drivers/ufs/host/cdns-pltfrm.c @@ -101,11 +101,10 @@ static void cdns_ufs_set_l4_attr(struct ufs_hba *hba) } /** - * cdns_ufs_set_hclkdiv() - * Sets HCLKDIV register value based on the core_clk + * cdns_ufs_set_hclkdiv() - set HCLKDIV register value based on the core_clk. * @hba: host controller instance * - * Return zero for success and non-zero for failure + * Return: zero for success and non-zero for failure. */ static int cdns_ufs_set_hclkdiv(struct ufs_hba *hba) { @@ -143,12 +142,11 @@ static int cdns_ufs_set_hclkdiv(struct ufs_hba *hba) } /** - * cdns_ufs_hce_enable_notify() - * Called before and after HCE enable bit is set. + * cdns_ufs_hce_enable_notify() - set HCLKDIV register * @hba: host controller instance * @status: notify stage (pre, post change) * - * Return zero for success and non-zero for failure + * Return: zero for success and non-zero for failure. */ static int cdns_ufs_hce_enable_notify(struct ufs_hba *hba, enum ufs_notify_change_status status) @@ -160,12 +158,10 @@ static int cdns_ufs_hce_enable_notify(struct ufs_hba *hba, } /** - * cdns_ufs_hibern8_notify() - * Called around hibern8 enter/exit. + * cdns_ufs_hibern8_notify() - save and restore L4 attributes. * @hba: host controller instance * @cmd: UIC Command * @status: notify stage (pre, post change) - * */ static void cdns_ufs_hibern8_notify(struct ufs_hba *hba, enum uic_cmd_dme cmd, enum ufs_notify_change_status status) @@ -177,12 +173,11 @@ static void cdns_ufs_hibern8_notify(struct ufs_hba *hba, enum uic_cmd_dme cmd, } /** - * cdns_ufs_link_startup_notify() - * Called before and after Link startup is carried out. + * cdns_ufs_link_startup_notify() - handle link startup. * @hba: host controller instance * @status: notify stage (pre, post change) * - * Return zero for success and non-zero for failure + * Return: zero for success and non-zero for failure. */ static int cdns_ufs_link_startup_notify(struct ufs_hba *hba, enum ufs_notify_change_status status) @@ -212,7 +207,7 @@ static int cdns_ufs_link_startup_notify(struct ufs_hba *hba, * cdns_ufs_init - performs additional ufs initialization * @hba: host controller instance * - * Returns status of initialization + * Return: status of initialization. */ static int cdns_ufs_init(struct ufs_hba *hba) { @@ -235,7 +230,7 @@ static int cdns_ufs_init(struct ufs_hba *hba) * cdns_ufs_m31_16nm_phy_initialization - performs m31 phy initialization * @hba: host controller instance * - * Always returns 0 + * Return: 0 (success). */ static int cdns_ufs_m31_16nm_phy_initialization(struct ufs_hba *hba) { @@ -284,7 +279,7 @@ MODULE_DEVICE_TABLE(of, cdns_ufs_of_match); * cdns_ufs_pltfrm_probe - probe routine of the driver * @pdev: pointer to platform device handle * - * Return zero for success and non-zero for failure + * Return: zero for success and non-zero for failure. */ static int cdns_ufs_pltfrm_probe(struct platform_device *pdev) { @@ -308,7 +303,7 @@ static int cdns_ufs_pltfrm_probe(struct platform_device *pdev) * cdns_ufs_pltfrm_remove - removes the ufs driver * @pdev: pointer to platform device handle * - * Always returns 0 + * Return: 0 (success). */ static int cdns_ufs_pltfrm_remove(struct platform_device *pdev) { diff --git a/drivers/ufs/host/tc-dwc-g210-pci.c b/drivers/ufs/host/tc-dwc-g210-pci.c index f96fe5855841..876781fd6861 100644 --- a/drivers/ufs/host/tc-dwc-g210-pci.c +++ b/drivers/ufs/host/tc-dwc-g210-pci.c @@ -51,7 +51,7 @@ static void tc_dwc_g210_pci_remove(struct pci_dev *pdev) * @pdev: pointer to PCI device handle * @id: PCI device id * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int tc_dwc_g210_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) diff --git a/drivers/ufs/host/tc-dwc-g210.c b/drivers/ufs/host/tc-dwc-g210.c index deb93dbd83a4..0ac53cc8465e 100644 --- a/drivers/ufs/host/tc-dwc-g210.c +++ b/drivers/ufs/host/tc-dwc-g210.c @@ -17,11 +17,10 @@ #include "tc-dwc-g210.h" /** - * tc_dwc_g210_setup_40bit_rmmi() - * This function configures Synopsys TC specific atributes (40-bit RMMI) + * tc_dwc_g210_setup_40bit_rmmi() - configure 40-bit RMMI. * @hba: Pointer to drivers structure * - * Returns 0 on success or non-zero value on failure + * Return: 0 on success or non-zero value on failure. */ static int tc_dwc_g210_setup_40bit_rmmi(struct ufs_hba *hba) { @@ -81,11 +80,10 @@ static int tc_dwc_g210_setup_40bit_rmmi(struct ufs_hba *hba) } /** - * tc_dwc_g210_setup_20bit_rmmi_lane0() - * This function configures Synopsys TC 20-bit RMMI Lane 0 + * tc_dwc_g210_setup_20bit_rmmi_lane0() - configure 20-bit RMMI Lane 0. * @hba: Pointer to drivers structure * - * Returns 0 on success or non-zero value on failure + * Return: 0 on success or non-zero value on failure. */ static int tc_dwc_g210_setup_20bit_rmmi_lane0(struct ufs_hba *hba) { @@ -134,11 +132,10 @@ static int tc_dwc_g210_setup_20bit_rmmi_lane0(struct ufs_hba *hba) } /** - * tc_dwc_g210_setup_20bit_rmmi_lane1() - * This function configures Synopsys TC 20-bit RMMI Lane 1 + * tc_dwc_g210_setup_20bit_rmmi_lane1() - configure 20-bit RMMI Lane 1. * @hba: Pointer to drivers structure * - * Returns 0 on success or non-zero value on failure + * Return: 0 on success or non-zero value on failure. */ static int tc_dwc_g210_setup_20bit_rmmi_lane1(struct ufs_hba *hba) { @@ -211,11 +208,10 @@ out: } /** - * tc_dwc_g210_setup_20bit_rmmi() - * This function configures Synopsys TC specific atributes (20-bit RMMI) + * tc_dwc_g210_setup_20bit_rmmi() - configure 20-bit RMMI. * @hba: Pointer to drivers structure * - * Returns 0 on success or non-zero value on failure + * Return: 0 on success or non-zero value on failure. */ static int tc_dwc_g210_setup_20bit_rmmi(struct ufs_hba *hba) { @@ -251,12 +247,10 @@ out: } /** - * tc_dwc_g210_config_40_bit() - * This function configures Local (host) Synopsys 40-bit TC specific attributes - * + * tc_dwc_g210_config_40_bit() - configure 40-bit TC specific attributes. * @hba: Pointer to drivers structure * - * Returns 0 on success non-zero value on failure + * Return: 0 on success non-zero value on failure. */ int tc_dwc_g210_config_40_bit(struct ufs_hba *hba) { @@ -283,12 +277,10 @@ out: EXPORT_SYMBOL(tc_dwc_g210_config_40_bit); /** - * tc_dwc_g210_config_20_bit() - * This function configures Local (host) Synopsys 20-bit TC specific attributes - * + * tc_dwc_g210_config_20_bit() - configure 20-bit TC specific attributes. * @hba: Pointer to drivers structure * - * Returns 0 on success non-zero value on failure + * Return: 0 on success non-zero value on failure. */ int tc_dwc_g210_config_20_bit(struct ufs_hba *hba) { diff --git a/drivers/ufs/host/ti-j721e-ufs.c b/drivers/ufs/host/ti-j721e-ufs.c index 122d650d0810..117eb7da92ac 100644 --- a/drivers/ufs/host/ti-j721e-ufs.c +++ b/drivers/ufs/host/ti-j721e-ufs.c @@ -81,6 +81,8 @@ static const struct of_device_id ti_j721e_ufs_of_match[] = { { }, }; +MODULE_DEVICE_TABLE(of, ti_j721e_ufs_of_match); + static struct platform_driver ti_j721e_ufs_driver = { .probe = ti_j721e_ufs_probe, .remove = ti_j721e_ufs_remove, diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c index e68b05976f9e..2383ecd88f1c 100644 --- a/drivers/ufs/host/ufs-mediatek.c +++ b/drivers/ufs/host/ufs-mediatek.c @@ -14,6 +14,7 @@ #include <linux/of.h> #include <linux/of_address.h> #include <linux/of_device.h> +#include <linux/of_platform.h> #include <linux/phy/phy.h> #include <linux/platform_device.h> #include <linux/pm_qos.h> @@ -27,8 +28,14 @@ #include <ufs/unipro.h> #include "ufs-mediatek.h" +static int ufs_mtk_config_mcq(struct ufs_hba *hba, bool irq); + #define CREATE_TRACE_POINTS #include "ufs-mediatek-trace.h" +#undef CREATE_TRACE_POINTS + +#define MAX_SUPP_MAC 64 +#define MCQ_QUEUE_OFFSET(c) ((((c) >> 16) & 0xFF) * 0x200) static const struct ufs_dev_quirk ufs_mtk_dev_fixups[] = { { .wmanufacturerid = UFS_ANY_VENDOR, @@ -659,7 +666,7 @@ static void ufs_mtk_pwr_ctrl(struct ufs_hba *hba, bool on) * @on: If true, enable clocks else disable them. * @status: PRE_CHANGE or POST_CHANGE notify * - * Returns 0 on success, non-zero on failure. + * Return: 0 on success, non-zero on failure. */ static int ufs_mtk_setup_clocks(struct ufs_hba *hba, bool on, enum ufs_notify_change_status status) @@ -840,6 +847,37 @@ static void ufs_mtk_vreg_fix_vccqx(struct ufs_hba *hba) } } +static void ufs_mtk_init_mcq_irq(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct platform_device *pdev; + int i; + int irq; + + host->mcq_nr_intr = UFSHCD_MAX_Q_NR; + pdev = container_of(hba->dev, struct platform_device, dev); + + for (i = 0; i < host->mcq_nr_intr; i++) { + /* irq index 0 is legacy irq, sq/cq irq start from index 1 */ + irq = platform_get_irq(pdev, i + 1); + if (irq < 0) { + host->mcq_intr_info[i].irq = MTK_MCQ_INVALID_IRQ; + goto failed; + } + host->mcq_intr_info[i].hba = hba; + host->mcq_intr_info[i].irq = irq; + dev_info(hba->dev, "get platform mcq irq: %d, %d\n", i, irq); + } + + return; +failed: + /* invalidate irq info */ + for (i = 0; i < host->mcq_nr_intr; i++) + host->mcq_intr_info[i].irq = MTK_MCQ_INVALID_IRQ; + + host->mcq_nr_intr = 0; +} + /** * ufs_mtk_init - find other essential mmio bases * @hba: host controller instance @@ -847,7 +885,7 @@ static void ufs_mtk_vreg_fix_vccqx(struct ufs_hba *hba) * Binds PHY with controller and powers up PHY enabling clocks * and regulators. * - * Returns -EPROBE_DEFER if binding fails, returns negative error + * Return: -EPROBE_DEFER if binding fails, returns negative error * on phy power up failure and returns zero on success. */ static int ufs_mtk_init(struct ufs_hba *hba) @@ -876,6 +914,8 @@ static int ufs_mtk_init(struct ufs_hba *hba) /* Initialize host capability */ ufs_mtk_init_host_caps(hba); + ufs_mtk_init_mcq_irq(hba); + err = ufs_mtk_bind_mphy(hba); if (err) goto out_variant_clear; @@ -1173,7 +1213,17 @@ static int ufs_mtk_link_set_hpm(struct ufs_hba *hba) else return err; - err = ufshcd_make_hba_operational(hba); + if (!hba->mcq_enabled) { + err = ufshcd_make_hba_operational(hba); + } else { + ufs_mtk_config_mcq(hba, false); + ufshcd_mcq_make_queues_operational(hba); + ufshcd_mcq_config_mac(hba, hba->nutrs); + /* Enable MCQ mode */ + ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x1, + REG_UFS_MEM_CFG); + } + if (err) return err; @@ -1497,6 +1547,121 @@ static int ufs_mtk_clk_scale_notify(struct ufs_hba *hba, bool scale_up, return 0; } +static int ufs_mtk_get_hba_mac(struct ufs_hba *hba) +{ + return MAX_SUPP_MAC; +} + +static int ufs_mtk_op_runtime_config(struct ufs_hba *hba) +{ + struct ufshcd_mcq_opr_info_t *opr; + int i; + + hba->mcq_opr[OPR_SQD].offset = REG_UFS_MTK_SQD; + hba->mcq_opr[OPR_SQIS].offset = REG_UFS_MTK_SQIS; + hba->mcq_opr[OPR_CQD].offset = REG_UFS_MTK_CQD; + hba->mcq_opr[OPR_CQIS].offset = REG_UFS_MTK_CQIS; + + for (i = 0; i < OPR_MAX; i++) { + opr = &hba->mcq_opr[i]; + opr->stride = REG_UFS_MCQ_STRIDE; + opr->base = hba->mmio_base + opr->offset; + } + + return 0; +} + +static int ufs_mtk_mcq_config_resource(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + /* fail mcq initialization if interrupt is not filled properly */ + if (!host->mcq_nr_intr) { + dev_info(hba->dev, "IRQs not ready. MCQ disabled."); + return -EINVAL; + } + + hba->mcq_base = hba->mmio_base + MCQ_QUEUE_OFFSET(hba->mcq_capabilities); + return 0; +} + +static irqreturn_t ufs_mtk_mcq_intr(int irq, void *__intr_info) +{ + struct ufs_mtk_mcq_intr_info *mcq_intr_info = __intr_info; + struct ufs_hba *hba = mcq_intr_info->hba; + struct ufs_hw_queue *hwq; + u32 events; + int qid = mcq_intr_info->qid; + + hwq = &hba->uhq[qid]; + + events = ufshcd_mcq_read_cqis(hba, qid); + if (events) + ufshcd_mcq_write_cqis(hba, events, qid); + + if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS) + ufshcd_mcq_poll_cqe_lock(hba, hwq); + + return IRQ_HANDLED; +} + +static int ufs_mtk_config_mcq_irq(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + u32 irq, i; + int ret; + + for (i = 0; i < host->mcq_nr_intr; i++) { + irq = host->mcq_intr_info[i].irq; + if (irq == MTK_MCQ_INVALID_IRQ) { + dev_err(hba->dev, "invalid irq. %d\n", i); + return -ENOPARAM; + } + + host->mcq_intr_info[i].qid = i; + ret = devm_request_irq(hba->dev, irq, ufs_mtk_mcq_intr, 0, UFSHCD, + &host->mcq_intr_info[i]); + + dev_dbg(hba->dev, "request irq %d intr %s\n", irq, ret ? "failed" : ""); + + if (ret) { + dev_err(hba->dev, "Cannot request irq %d\n", ret); + return ret; + } + } + + return 0; +} + +static int ufs_mtk_config_mcq(struct ufs_hba *hba, bool irq) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + int ret = 0; + + if (!host->mcq_set_intr) { + /* Disable irq option register */ + ufshcd_rmwl(hba, MCQ_INTR_EN_MSK, 0, REG_UFS_MMIO_OPT_CTRL_0); + + if (irq) { + ret = ufs_mtk_config_mcq_irq(hba); + if (ret) + return ret; + } + + host->mcq_set_intr = true; + } + + ufshcd_rmwl(hba, MCQ_AH8, MCQ_AH8, REG_UFS_MMIO_OPT_CTRL_0); + ufshcd_rmwl(hba, MCQ_INTR_EN_MSK, MCQ_MULTI_INTR_EN, REG_UFS_MMIO_OPT_CTRL_0); + + return 0; +} + +static int ufs_mtk_config_esi(struct ufs_hba *hba) +{ + return ufs_mtk_config_mcq(hba, true); +} + /* * struct ufs_hba_mtk_vops - UFS MTK specific variant operations * @@ -1520,13 +1685,18 @@ static const struct ufs_hba_variant_ops ufs_hba_mtk_vops = { .event_notify = ufs_mtk_event_notify, .config_scaling_param = ufs_mtk_config_scaling_param, .clk_scale_notify = ufs_mtk_clk_scale_notify, + /* mcq vops */ + .get_hba_mac = ufs_mtk_get_hba_mac, + .op_runtime_config = ufs_mtk_op_runtime_config, + .mcq_config_resource = ufs_mtk_mcq_config_resource, + .config_esi = ufs_mtk_config_esi, }; /** * ufs_mtk_probe - probe routine of the driver * @pdev: pointer to Platform device handle * - * Return zero for success and non-zero for failure + * Return: zero for success and non-zero for failure. */ static int ufs_mtk_probe(struct platform_device *pdev) { @@ -1566,7 +1736,7 @@ skip_reset: out: if (err) - dev_info(dev, "probe failed %d\n", err); + dev_err(dev, "probe failed %d\n", err); of_node_put(reset_node); return err; diff --git a/drivers/ufs/host/ufs-mediatek.h b/drivers/ufs/host/ufs-mediatek.h index 2fc6d7b87694..f76e80d91729 100644 --- a/drivers/ufs/host/ufs-mediatek.h +++ b/drivers/ufs/host/ufs-mediatek.h @@ -11,10 +11,26 @@ #include <linux/soc/mediatek/mtk_sip_svc.h> /* + * MCQ define and struct + */ +#define UFSHCD_MAX_Q_NR 8 +#define MTK_MCQ_INVALID_IRQ 0xFFFF + +/* REG_UFS_MMIO_OPT_CTRL_0 160h */ +#define EHS_EN BIT(0) +#define PFM_IMPV BIT(1) +#define MCQ_MULTI_INTR_EN BIT(2) +#define MCQ_CMB_INTR_EN BIT(3) +#define MCQ_AH8 BIT(4) + +#define MCQ_INTR_EN_MSK (MCQ_MULTI_INTR_EN | MCQ_CMB_INTR_EN) + +/* * Vendor specific UFSHCI Registers */ #define REG_UFS_XOUFS_CTRL 0x140 #define REG_UFS_REFCLK_CTRL 0x144 +#define REG_UFS_MMIO_OPT_CTRL_0 0x160 #define REG_UFS_EXTREG 0x2100 #define REG_UFS_MPHYCTRL 0x2200 #define REG_UFS_MTK_IP_VER 0x2240 @@ -26,6 +42,13 @@ #define REG_UFS_DEBUG_SEL_B2 0x22D8 #define REG_UFS_DEBUG_SEL_B3 0x22DC +#define REG_UFS_MTK_SQD 0x2800 +#define REG_UFS_MTK_SQIS 0x2814 +#define REG_UFS_MTK_CQD 0x281C +#define REG_UFS_MTK_CQIS 0x2824 + +#define REG_UFS_MCQ_STRIDE 0x30 + /* * Ref-clk control * @@ -136,6 +159,12 @@ struct ufs_mtk_hw_ver { u8 major; }; +struct ufs_mtk_mcq_intr_info { + struct ufs_hba *hba; + u32 irq; + u8 qid; +}; + struct ufs_mtk_host { struct phy *mphy; struct pm_qos_request pm_qos_req; @@ -155,6 +184,10 @@ struct ufs_mtk_host { u16 ref_clk_ungating_wait_us; u16 ref_clk_gating_wait_us; u32 ip_ver; + + bool mcq_set_intr; + int mcq_nr_intr; + struct ufs_mtk_mcq_intr_info mcq_intr_info[UFSHCD_MAX_Q_NR]; }; /* diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c index c1557d21b027..d1149b1c3ed5 100644 --- a/drivers/ufs/host/ufs-qcom.c +++ b/drivers/ufs/host/ufs-qcom.c @@ -7,6 +7,7 @@ #include <linux/time.h> #include <linux/clk.h> #include <linux/delay.h> +#include <linux/interconnect.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> @@ -46,6 +47,49 @@ enum { TSTBUS_MAX, }; +#define QCOM_UFS_MAX_GEAR 4 +#define QCOM_UFS_MAX_LANE 2 + +enum { + MODE_MIN, + MODE_PWM, + MODE_HS_RA, + MODE_HS_RB, + MODE_MAX, +}; + +static const struct __ufs_qcom_bw_table { + u32 mem_bw; + u32 cfg_bw; +} ufs_qcom_bw_table[MODE_MAX + 1][QCOM_UFS_MAX_GEAR + 1][QCOM_UFS_MAX_LANE + 1] = { + [MODE_MIN][0][0] = { 0, 0 }, /* Bandwidth values in KB/s */ + [MODE_PWM][UFS_PWM_G1][UFS_LANE_1] = { 922, 1000 }, + [MODE_PWM][UFS_PWM_G2][UFS_LANE_1] = { 1844, 1000 }, + [MODE_PWM][UFS_PWM_G3][UFS_LANE_1] = { 3688, 1000 }, + [MODE_PWM][UFS_PWM_G4][UFS_LANE_1] = { 7376, 1000 }, + [MODE_PWM][UFS_PWM_G1][UFS_LANE_2] = { 1844, 1000 }, + [MODE_PWM][UFS_PWM_G2][UFS_LANE_2] = { 3688, 1000 }, + [MODE_PWM][UFS_PWM_G3][UFS_LANE_2] = { 7376, 1000 }, + [MODE_PWM][UFS_PWM_G4][UFS_LANE_2] = { 14752, 1000 }, + [MODE_HS_RA][UFS_HS_G1][UFS_LANE_1] = { 127796, 1000 }, + [MODE_HS_RA][UFS_HS_G2][UFS_LANE_1] = { 255591, 1000 }, + [MODE_HS_RA][UFS_HS_G3][UFS_LANE_1] = { 1492582, 102400 }, + [MODE_HS_RA][UFS_HS_G4][UFS_LANE_1] = { 2915200, 204800 }, + [MODE_HS_RA][UFS_HS_G1][UFS_LANE_2] = { 255591, 1000 }, + [MODE_HS_RA][UFS_HS_G2][UFS_LANE_2] = { 511181, 1000 }, + [MODE_HS_RA][UFS_HS_G3][UFS_LANE_2] = { 1492582, 204800 }, + [MODE_HS_RA][UFS_HS_G4][UFS_LANE_2] = { 2915200, 409600 }, + [MODE_HS_RB][UFS_HS_G1][UFS_LANE_1] = { 149422, 1000 }, + [MODE_HS_RB][UFS_HS_G2][UFS_LANE_1] = { 298189, 1000 }, + [MODE_HS_RB][UFS_HS_G3][UFS_LANE_1] = { 1492582, 102400 }, + [MODE_HS_RB][UFS_HS_G4][UFS_LANE_1] = { 2915200, 204800 }, + [MODE_HS_RB][UFS_HS_G1][UFS_LANE_2] = { 298189, 1000 }, + [MODE_HS_RB][UFS_HS_G2][UFS_LANE_2] = { 596378, 1000 }, + [MODE_HS_RB][UFS_HS_G3][UFS_LANE_2] = { 1492582, 204800 }, + [MODE_HS_RB][UFS_HS_G4][UFS_LANE_2] = { 2915200, 409600 }, + [MODE_MAX][0][0] = { 7643136, 307200 }, +}; + static struct ufs_qcom_host *ufs_qcom_hosts[MAX_UFS_QCOM_HOSTS]; static void ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host *host); @@ -485,7 +529,7 @@ static int ufs_qcom_hce_enable_notify(struct ufs_hba *hba, } /* - * Returns zero for success and non-zero in case of a failure + * Return: zero for success and non-zero in case of a failure. */ static int ufs_qcom_cfg_timers(struct ufs_hba *hba, u32 gear, u32 hs, u32 rate, bool update_link_startup_timer) @@ -789,6 +833,51 @@ static void ufs_qcom_dev_ref_clk_ctrl(struct ufs_qcom_host *host, bool enable) } } +static int ufs_qcom_icc_set_bw(struct ufs_qcom_host *host, u32 mem_bw, u32 cfg_bw) +{ + struct device *dev = host->hba->dev; + int ret; + + ret = icc_set_bw(host->icc_ddr, 0, mem_bw); + if (ret < 0) { + dev_err(dev, "failed to set bandwidth request: %d\n", ret); + return ret; + } + + ret = icc_set_bw(host->icc_cpu, 0, cfg_bw); + if (ret < 0) { + dev_err(dev, "failed to set bandwidth request: %d\n", ret); + return ret; + } + + return 0; +} + +static struct __ufs_qcom_bw_table ufs_qcom_get_bw_table(struct ufs_qcom_host *host) +{ + struct ufs_pa_layer_attr *p = &host->dev_req_params; + int gear = max_t(u32, p->gear_rx, p->gear_tx); + int lane = max_t(u32, p->lane_rx, p->lane_tx); + + if (ufshcd_is_hs_mode(p)) { + if (p->hs_rate == PA_HS_MODE_B) + return ufs_qcom_bw_table[MODE_HS_RB][gear][lane]; + else + return ufs_qcom_bw_table[MODE_HS_RA][gear][lane]; + } else { + return ufs_qcom_bw_table[MODE_PWM][gear][lane]; + } +} + +static int ufs_qcom_icc_update_bw(struct ufs_qcom_host *host) +{ + struct __ufs_qcom_bw_table bw_table; + + bw_table = ufs_qcom_get_bw_table(host); + + return ufs_qcom_icc_set_bw(host, bw_table.mem_bw, bw_table.cfg_bw); +} + static int ufs_qcom_pwr_change_notify(struct ufs_hba *hba, enum ufs_notify_change_status status, struct ufs_pa_layer_attr *dev_max_params, @@ -852,6 +941,8 @@ static int ufs_qcom_pwr_change_notify(struct ufs_hba *hba, memcpy(&host->dev_req_params, dev_req_params, sizeof(*dev_req_params)); + ufs_qcom_icc_update_bw(host); + /* disable the device ref clock if entered PWM mode */ if (ufshcd_is_hs_mode(&hba->pwr_info) && !ufshcd_is_hs_mode(dev_req_params)) @@ -964,7 +1055,7 @@ static void ufs_qcom_set_caps(struct ufs_hba *hba) * @on: If true, enable clocks else disable them. * @status: PRE_CHANGE or POST_CHANGE notify * - * Returns 0 on success, non-zero on failure. + * Return: 0 on success, non-zero on failure. */ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on, enum ufs_notify_change_status status) @@ -981,7 +1072,9 @@ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on, switch (status) { case PRE_CHANGE: - if (!on) { + if (on) { + ufs_qcom_icc_update_bw(host); + } else { if (!ufs_qcom_is_link_active(hba)) { /* disable device ref_clk */ ufs_qcom_dev_ref_clk_ctrl(host, false); @@ -993,6 +1086,9 @@ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on, /* enable the device ref clock for HS mode*/ if (ufshcd_is_hs_mode(&hba->pwr_info)) ufs_qcom_dev_ref_clk_ctrl(host, true); + } else { + ufs_qcom_icc_set_bw(host, ufs_qcom_bw_table[MODE_MIN][0][0].mem_bw, + ufs_qcom_bw_table[MODE_MIN][0][0].cfg_bw); } break; } @@ -1031,6 +1127,34 @@ static const struct reset_control_ops ufs_qcom_reset_ops = { .deassert = ufs_qcom_reset_deassert, }; +static int ufs_qcom_icc_init(struct ufs_qcom_host *host) +{ + struct device *dev = host->hba->dev; + int ret; + + host->icc_ddr = devm_of_icc_get(dev, "ufs-ddr"); + if (IS_ERR(host->icc_ddr)) + return dev_err_probe(dev, PTR_ERR(host->icc_ddr), + "failed to acquire interconnect path\n"); + + host->icc_cpu = devm_of_icc_get(dev, "cpu-ufs"); + if (IS_ERR(host->icc_cpu)) + return dev_err_probe(dev, PTR_ERR(host->icc_cpu), + "failed to acquire interconnect path\n"); + + /* + * Set Maximum bandwidth vote before initializing the UFS controller and + * device. Ideally, a minimal interconnect vote would suffice for the + * initialization, but a max vote would allow faster initialization. + */ + ret = ufs_qcom_icc_set_bw(host, ufs_qcom_bw_table[MODE_MAX][0][0].mem_bw, + ufs_qcom_bw_table[MODE_MAX][0][0].cfg_bw); + if (ret < 0) + return dev_err_probe(dev, ret, "failed to set bandwidth request\n"); + + return 0; +} + /** * ufs_qcom_init - bind phy with controller * @hba: host controller instance @@ -1038,7 +1162,7 @@ static const struct reset_control_ops ufs_qcom_reset_ops = { * Binds PHY with controller and powers up PHY enabling clocks * and regulators. * - * Returns -EPROBE_DEFER if binding fails, returns negative error + * Return: -EPROBE_DEFER if binding fails, returns negative error * on phy power up failure and returns zero on success. */ static int ufs_qcom_init(struct ufs_hba *hba) @@ -1085,6 +1209,10 @@ static int ufs_qcom_init(struct ufs_hba *hba) } } + err = ufs_qcom_icc_init(host); + if (err) + goto out_variant_clear; + host->device_reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); if (IS_ERR(host->device_reset)) { @@ -1254,6 +1382,10 @@ static int ufs_qcom_clk_scale_notify(struct ufs_hba *hba, struct ufs_pa_layer_attr *dev_req_params = &host->dev_req_params; int err = 0; + /* check the host controller state before sending hibern8 cmd */ + if (!ufshcd_is_hba_active(hba)) + return 0; + if (status == PRE_CHANGE) { err = ufshcd_uic_hibern8_enter(hba); if (err) @@ -1282,6 +1414,7 @@ static int ufs_qcom_clk_scale_notify(struct ufs_hba *hba, dev_req_params->pwr_rx, dev_req_params->hs_rate, false); + ufs_qcom_icc_update_bw(host); ufshcd_uic_hibern8_exit(hba); } @@ -1483,6 +1616,7 @@ static void ufs_qcom_config_scaling_param(struct ufs_hba *hba, struct devfreq_simple_ondemand_data *d) { p->polling_ms = 60; + p->timer = DEVFREQ_TIMER_DELAYED; d->upthreshold = 70; d->downdifferential = 5; } @@ -1643,11 +1777,12 @@ static void ufs_qcom_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) ufshcd_mcq_config_esi(hba, msg); } -static irqreturn_t ufs_qcom_mcq_esi_handler(int irq, void *__hba) +static irqreturn_t ufs_qcom_mcq_esi_handler(int irq, void *data) { - struct ufs_hba *hba = __hba; - struct ufs_qcom_host *host = ufshcd_get_variant(hba); - u32 id = irq - host->esi_base; + struct msi_desc *desc = data; + struct device *dev = msi_desc_to_dev(desc); + struct ufs_hba *hba = dev_get_drvdata(dev); + u32 id = desc->msi_index; struct ufs_hw_queue *hwq = &hba->uhq[id]; ufshcd_mcq_write_cqis(hba, 0x1, id); @@ -1665,8 +1800,6 @@ static int ufs_qcom_config_esi(struct ufs_hba *hba) if (host->esi_enabled) return 0; - else if (host->esi_base < 0) - return -EINVAL; /* * 1. We only handle CQs as of now. @@ -1675,16 +1808,16 @@ static int ufs_qcom_config_esi(struct ufs_hba *hba) nr_irqs = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]; ret = platform_msi_domain_alloc_irqs(hba->dev, nr_irqs, ufs_qcom_write_msi_msg); - if (ret) + if (ret) { + dev_err(hba->dev, "Failed to request Platform MSI %d\n", ret); goto out; + } + msi_lock_descs(hba->dev); msi_for_each_desc(desc, hba->dev, MSI_DESC_ALL) { - if (!desc->msi_index) - host->esi_base = desc->irq; - ret = devm_request_irq(hba->dev, desc->irq, ufs_qcom_mcq_esi_handler, - IRQF_SHARED, "qcom-mcq-esi", hba); + IRQF_SHARED, "qcom-mcq-esi", desc); if (ret) { dev_err(hba->dev, "%s: Fail to request IRQ for %d, err = %d\n", __func__, desc->irq, ret); @@ -1692,14 +1825,17 @@ static int ufs_qcom_config_esi(struct ufs_hba *hba) break; } } + msi_unlock_descs(hba->dev); if (ret) { /* Rewind */ + msi_lock_descs(hba->dev); msi_for_each_desc(desc, hba->dev, MSI_DESC_ALL) { if (desc == failed_desc) break; devm_free_irq(hba->dev, desc->irq, hba); } + msi_unlock_descs(hba->dev); platform_msi_domain_free_irqs(hba->dev); } else { if (host->hw_ver.major == 6 && host->hw_ver.minor == 0 && @@ -1712,12 +1848,8 @@ static int ufs_qcom_config_esi(struct ufs_hba *hba) } out: - if (ret) { - host->esi_base = -1; - dev_warn(hba->dev, "Failed to request Platform MSI %d\n", ret); - } else { + if (!ret) host->esi_enabled = true; - } return ret; } @@ -1757,7 +1889,7 @@ static const struct ufs_hba_variant_ops ufs_hba_qcom_vops = { * ufs_qcom_probe - probe routine of the driver * @pdev: pointer to Platform device handle * - * Return zero for success and non-zero for failure + * Return: zero for success and non-zero for failure. */ static int ufs_qcom_probe(struct platform_device *pdev) { diff --git a/drivers/ufs/host/ufs-qcom.h b/drivers/ufs/host/ufs-qcom.h index 6289ad5a42d0..d6f8e74bd538 100644 --- a/drivers/ufs/host/ufs-qcom.h +++ b/drivers/ufs/host/ufs-qcom.h @@ -206,6 +206,9 @@ struct ufs_qcom_host { struct clk *tx_l1_sync_clk; bool is_lane_clks_enabled; + struct icc_path *icc_ddr; + struct icc_path *icc_cpu; + #ifdef CONFIG_SCSI_UFS_CRYPTO struct qcom_ice *ice; #endif @@ -226,7 +229,6 @@ struct ufs_qcom_host { u32 hs_gear; - int esi_base; bool esi_enabled; }; diff --git a/drivers/ufs/host/ufs-renesas.c b/drivers/ufs/host/ufs-renesas.c index ab0652d8705a..cc94970b86c9 100644 --- a/drivers/ufs/host/ufs-renesas.c +++ b/drivers/ufs/host/ufs-renesas.c @@ -12,7 +12,7 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <ufs/ufshcd.h> diff --git a/drivers/ufs/host/ufshcd-dwc.c b/drivers/ufs/host/ufshcd-dwc.c index e28a67e1e314..21b1cf912dcc 100644 --- a/drivers/ufs/host/ufshcd-dwc.c +++ b/drivers/ufs/host/ufshcd-dwc.c @@ -34,9 +34,7 @@ int ufshcd_dwc_dme_set_attrs(struct ufs_hba *hba, EXPORT_SYMBOL(ufshcd_dwc_dme_set_attrs); /** - * ufshcd_dwc_program_clk_div() - * This function programs the clk divider value. This value is needed to - * provide 1 microsecond tick to unipro layer. + * ufshcd_dwc_program_clk_div() - program clock divider. * @hba: Private Structure pointer * @divider_val: clock divider value to be programmed * @@ -47,11 +45,10 @@ static void ufshcd_dwc_program_clk_div(struct ufs_hba *hba, u32 divider_val) } /** - * ufshcd_dwc_link_is_up() - * Check if link is up + * ufshcd_dwc_link_is_up() - check if link is up. * @hba: private structure pointer * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int ufshcd_dwc_link_is_up(struct ufs_hba *hba) { @@ -68,7 +65,9 @@ static int ufshcd_dwc_link_is_up(struct ufs_hba *hba) } /** - * ufshcd_dwc_connection_setup() + * ufshcd_dwc_connection_setup() - configure unipro attributes. + * @hba: pointer to drivers private data + * * This function configures both the local side (host) and the peer side * (device) unipro attributes to establish the connection to application/ * cport. @@ -76,9 +75,7 @@ static int ufshcd_dwc_link_is_up(struct ufs_hba *hba) * have this connection setup on reset. But invoking this function does no * harm and should be fine even working with any ufs device. * - * @hba: pointer to drivers private data - * - * Returns 0 on success non-zero value on failure + * Return: 0 on success non-zero value on failure. */ static int ufshcd_dwc_connection_setup(struct ufs_hba *hba) { @@ -107,12 +104,11 @@ static int ufshcd_dwc_connection_setup(struct ufs_hba *hba) } /** - * ufshcd_dwc_link_startup_notify() - * UFS Host DWC specific link startup sequence + * ufshcd_dwc_link_startup_notify() - program clock divider. * @hba: private structure pointer * @status: Callback notify status * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ int ufshcd_dwc_link_startup_notify(struct ufs_hba *hba, enum ufs_notify_change_status status) diff --git a/drivers/ufs/host/ufshcd-pci.c b/drivers/ufs/host/ufshcd-pci.c index cf3987773051..248a49e5e7f3 100644 --- a/drivers/ufs/host/ufshcd-pci.c +++ b/drivers/ufs/host/ufshcd-pci.c @@ -524,7 +524,7 @@ static void ufshcd_pci_remove(struct pci_dev *pdev) * @pdev: pointer to PCI device handle * @id: PCI device id * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int ufshcd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) @@ -590,6 +590,7 @@ static const struct dev_pm_ops ufshcd_pci_pm_ops = { }; static const struct pci_device_id ufshcd_pci_tbl[] = { + { PCI_VENDOR_ID_REDHAT, 0x0013, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, { PCI_VENDOR_ID_SAMSUNG, 0xC00C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 }, { PCI_VDEVICE(INTEL, 0x9DFA), (kernel_ulong_t)&ufs_intel_cnl_hba_vops }, { PCI_VDEVICE(INTEL, 0x4B41), (kernel_ulong_t)&ufs_intel_ehl_hba_vops }, diff --git a/drivers/ufs/host/ufshcd-pltfrm.c b/drivers/ufs/host/ufshcd-pltfrm.c index 0b7430033047..797a4dfe45d9 100644 --- a/drivers/ufs/host/ufshcd-pltfrm.c +++ b/drivers/ufs/host/ufshcd-pltfrm.c @@ -166,6 +166,8 @@ EXPORT_SYMBOL_GPL(ufshcd_populate_vreg); * If any of the supplies are not defined it is assumed that they are always-on * and hence return zero. If the property is defined but parsing is failed * then return corresponding error. + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_parse_regulator_info(struct ufs_hba *hba) { @@ -212,7 +214,7 @@ static void ufshcd_init_lanes_per_dir(struct ufs_hba *hba) * @dev_max: pointer to device attributes * @agreed_pwr: returned agreed attributes * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ int ufshcd_get_pwr_dev_param(const struct ufs_dev_params *pltfrm_param, const struct ufs_pa_layer_attr *dev_max, @@ -305,8 +307,8 @@ EXPORT_SYMBOL_GPL(ufshcd_get_pwr_dev_param); void ufshcd_init_pwr_dev_param(struct ufs_dev_params *dev_param) { *dev_param = (struct ufs_dev_params){ - .tx_lanes = 2, - .rx_lanes = 2, + .tx_lanes = UFS_LANE_2, + .rx_lanes = UFS_LANE_2, .hs_rx_gear = UFS_HS_G3, .hs_tx_gear = UFS_HS_G3, .pwm_rx_gear = UFS_PWM_G4, @@ -326,7 +328,7 @@ EXPORT_SYMBOL_GPL(ufshcd_init_pwr_dev_param); * @pdev: pointer to Platform device handle * @vops: pointer to variant ops * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ int ufshcd_pltfrm_init(struct platform_device *pdev, const struct ufs_hba_variant_ops *vops) @@ -373,7 +375,8 @@ int ufshcd_pltfrm_init(struct platform_device *pdev, err = ufshcd_init(hba, mmio_base, irq); if (err) { - dev_err(dev, "Initialization failed\n"); + dev_err_probe(dev, err, "Initialization failed with error %d\n", + err); goto dealloc_host; } |