summaryrefslogtreecommitdiff
path: root/drivers/scsi/ufs
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/scsi/ufs')
-rw-r--r--drivers/scsi/ufs/Kconfig1
-rw-r--r--drivers/scsi/ufs/ufs-exynos.c13
-rw-r--r--drivers/scsi/ufs/ufs-mediatek.c266
-rw-r--r--drivers/scsi/ufs/ufs-mediatek.h29
-rw-r--r--drivers/scsi/ufs/ufs-qcom.c262
-rw-r--r--drivers/scsi/ufs/ufs-qcom.h11
-rw-r--r--drivers/scsi/ufs/ufs-sysfs.c10
-rw-r--r--drivers/scsi/ufs/ufshcd-crypto.c4
-rw-r--r--drivers/scsi/ufs/ufshcd-pci.c127
-rw-r--r--drivers/scsi/ufs/ufshcd.c846
-rw-r--r--drivers/scsi/ufs/ufshcd.h22
-rw-r--r--drivers/scsi/ufs/ufshci.h1
-rw-r--r--drivers/scsi/ufs/unipro.h3
13 files changed, 979 insertions, 616 deletions
diff --git a/drivers/scsi/ufs/Kconfig b/drivers/scsi/ufs/Kconfig
index f6394999b98c..dcdb4eb1f90b 100644
--- a/drivers/scsi/ufs/Kconfig
+++ b/drivers/scsi/ufs/Kconfig
@@ -165,7 +165,6 @@ config SCSI_UFS_BSG
config SCSI_UFS_EXYNOS
tristate "EXYNOS specific hooks to UFS controller platform driver"
depends on SCSI_UFSHCD_PLATFORM && (ARCH_EXYNOS || COMPILE_TEST)
- select PHY_SAMSUNG_UFS
help
This selects the EXYNOS specific additions to UFSHCD platform driver.
UFS host on EXYNOS includes HCI and UNIPRO layer, and associates with
diff --git a/drivers/scsi/ufs/ufs-exynos.c b/drivers/scsi/ufs/ufs-exynos.c
index 8f1b6f61a776..5e6b95dbb578 100644
--- a/drivers/scsi/ufs/ufs-exynos.c
+++ b/drivers/scsi/ufs/ufs-exynos.c
@@ -940,7 +940,6 @@ static int exynos_ufs_init(struct ufs_hba *hba)
struct device *dev = hba->dev;
struct platform_device *pdev = to_platform_device(dev);
struct exynos_ufs *ufs;
- struct resource *res;
int ret;
ufs = devm_kzalloc(dev, sizeof(*ufs), GFP_KERNEL);
@@ -948,24 +947,21 @@ static int exynos_ufs_init(struct ufs_hba *hba)
return -ENOMEM;
/* exynos-specific hci */
- res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "vs_hci");
- ufs->reg_hci = devm_ioremap_resource(dev, res);
+ ufs->reg_hci = devm_platform_ioremap_resource_byname(pdev, "vs_hci");
if (IS_ERR(ufs->reg_hci)) {
dev_err(dev, "cannot ioremap for hci vendor register\n");
return PTR_ERR(ufs->reg_hci);
}
/* unipro */
- res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "unipro");
- ufs->reg_unipro = devm_ioremap_resource(dev, res);
+ ufs->reg_unipro = devm_platform_ioremap_resource_byname(pdev, "unipro");
if (IS_ERR(ufs->reg_unipro)) {
dev_err(dev, "cannot ioremap for unipro register\n");
return PTR_ERR(ufs->reg_unipro);
}
/* ufs protector */
- res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ufsp");
- ufs->reg_ufsp = devm_ioremap_resource(dev, res);
+ ufs->reg_ufsp = devm_platform_ioremap_resource_byname(pdev, "ufsp");
if (IS_ERR(ufs->reg_ufsp)) {
dev_err(dev, "cannot ioremap for ufs protector register\n");
return PTR_ERR(ufs->reg_ufsp);
@@ -1252,7 +1248,8 @@ struct exynos_ufs_drv_data exynos_ufs_drvs = {
UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR |
UFSHCI_QUIRK_BROKEN_HCE |
UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR |
- UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR,
+ UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR |
+ UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL,
.opts = EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL |
EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL |
EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX |
diff --git a/drivers/scsi/ufs/ufs-mediatek.c b/drivers/scsi/ufs/ufs-mediatek.c
index 1755dd6b04ae..8df73bc2f8cb 100644
--- a/drivers/scsi/ufs/ufs-mediatek.c
+++ b/drivers/scsi/ufs/ufs-mediatek.c
@@ -10,9 +10,11 @@
#include <linux/bitfield.h>
#include <linux/of.h>
#include <linux/of_address.h>
+#include <linux/of_device.h>
#include <linux/phy/phy.h>
#include <linux/platform_device.h>
#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
#include <linux/soc/mediatek/mtk_sip_svc.h>
#include "ufshcd.h"
@@ -43,6 +45,28 @@ static struct ufs_dev_fix ufs_mtk_dev_fixups[] = {
END_FIX
};
+static const struct ufs_mtk_host_cfg ufs_mtk_mt8192_cfg = {
+ .caps = UFS_MTK_CAP_BOOST_CRYPT_ENGINE,
+};
+
+static const struct of_device_id ufs_mtk_of_match[] = {
+ {
+ .compatible = "mediatek,mt8183-ufshci",
+ },
+ {
+ .compatible = "mediatek,mt8192-ufshci",
+ .data = &ufs_mtk_mt8192_cfg
+ },
+ {},
+};
+
+static bool ufs_mtk_is_boost_crypt_enabled(struct ufs_hba *hba)
+{
+ struct ufs_mtk_host *host = ufshcd_get_variant(hba);
+
+ return (host->caps & UFS_MTK_CAP_BOOST_CRYPT_ENGINE);
+}
+
static void ufs_mtk_cfg_unipro_cg(struct ufs_hba *hba, bool enable)
{
u32 tmp;
@@ -91,16 +115,57 @@ static void ufs_mtk_crypto_enable(struct ufs_hba *hba)
}
}
+static void ufs_mtk_host_reset(struct ufs_hba *hba)
+{
+ struct ufs_mtk_host *host = ufshcd_get_variant(hba);
+
+ reset_control_assert(host->hci_reset);
+ reset_control_assert(host->crypto_reset);
+ reset_control_assert(host->unipro_reset);
+
+ usleep_range(100, 110);
+
+ reset_control_deassert(host->unipro_reset);
+ reset_control_deassert(host->crypto_reset);
+ reset_control_deassert(host->hci_reset);
+}
+
+static void ufs_mtk_init_reset_control(struct ufs_hba *hba,
+ struct reset_control **rc,
+ char *str)
+{
+ *rc = devm_reset_control_get(hba->dev, str);
+ if (IS_ERR(*rc)) {
+ dev_info(hba->dev, "Failed to get reset control %s: %ld\n",
+ str, PTR_ERR(*rc));
+ *rc = NULL;
+ }
+}
+
+static void ufs_mtk_init_reset(struct ufs_hba *hba)
+{
+ struct ufs_mtk_host *host = ufshcd_get_variant(hba);
+
+ ufs_mtk_init_reset_control(hba, &host->hci_reset,
+ "hci_rst");
+ ufs_mtk_init_reset_control(hba, &host->unipro_reset,
+ "unipro_rst");
+ ufs_mtk_init_reset_control(hba, &host->crypto_reset,
+ "crypto_rst");
+}
+
static int ufs_mtk_hce_enable_notify(struct ufs_hba *hba,
enum ufs_notify_change_status status)
{
struct ufs_mtk_host *host = ufshcd_get_variant(hba);
if (status == PRE_CHANGE) {
- if (host->unipro_lpm)
+ if (host->unipro_lpm) {
hba->vps->hba_enable_delay_us = 0;
- else
+ } else {
hba->vps->hba_enable_delay_us = 600;
+ ufs_mtk_host_reset(hba);
+ }
if (hba->caps & UFSHCD_CAP_CRYPTO)
ufs_mtk_crypto_enable(hba);
@@ -129,7 +194,10 @@ static int ufs_mtk_bind_mphy(struct ufs_hba *hba)
__func__, err);
} else if (IS_ERR(host->mphy)) {
err = PTR_ERR(host->mphy);
- dev_info(dev, "%s: PHY get failed %d\n", __func__, err);
+ if (err != -ENODEV) {
+ dev_info(dev, "%s: PHY get failed %d\n", __func__,
+ err);
+ }
}
if (err)
@@ -249,6 +317,144 @@ static void ufs_mtk_mphy_power_on(struct ufs_hba *hba, bool on)
host->mphy_powered_on = on;
}
+static int ufs_mtk_get_host_clk(struct device *dev, const char *name,
+ struct clk **clk_out)
+{
+ struct clk *clk;
+ int err = 0;
+
+ clk = devm_clk_get(dev, name);
+ if (IS_ERR(clk))
+ err = PTR_ERR(clk);
+ else
+ *clk_out = clk;
+
+ return err;
+}
+
+static void ufs_mtk_boost_crypt(struct ufs_hba *hba, bool boost)
+{
+ struct ufs_mtk_host *host = ufshcd_get_variant(hba);
+ struct ufs_mtk_crypt_cfg *cfg;
+ struct regulator *reg;
+ int volt, ret;
+
+ if (!ufs_mtk_is_boost_crypt_enabled(hba))
+ return;
+
+ cfg = host->crypt;
+ volt = cfg->vcore_volt;
+ reg = cfg->reg_vcore;
+
+ ret = clk_prepare_enable(cfg->clk_crypt_mux);
+ if (ret) {
+ dev_info(hba->dev, "clk_prepare_enable(): %d\n",
+ ret);
+ return;
+ }
+
+ if (boost) {
+ ret = regulator_set_voltage(reg, volt, INT_MAX);
+ if (ret) {
+ dev_info(hba->dev,
+ "failed to set vcore to %d\n", volt);
+ goto out;
+ }
+
+ ret = clk_set_parent(cfg->clk_crypt_mux,
+ cfg->clk_crypt_perf);
+ if (ret) {
+ dev_info(hba->dev,
+ "failed to set clk_crypt_perf\n");
+ regulator_set_voltage(reg, 0, INT_MAX);
+ goto out;
+ }
+ } else {
+ ret = clk_set_parent(cfg->clk_crypt_mux,
+ cfg->clk_crypt_lp);
+ if (ret) {
+ dev_info(hba->dev,
+ "failed to set clk_crypt_lp\n");
+ goto out;
+ }
+
+ ret = regulator_set_voltage(reg, 0, INT_MAX);
+ if (ret) {
+ dev_info(hba->dev,
+ "failed to set vcore to MIN\n");
+ }
+ }
+out:
+ clk_disable_unprepare(cfg->clk_crypt_mux);
+}
+
+static int ufs_mtk_init_host_clk(struct ufs_hba *hba, const char *name,
+ struct clk **clk)
+{
+ int ret;
+
+ ret = ufs_mtk_get_host_clk(hba->dev, name, clk);
+ if (ret) {
+ dev_info(hba->dev, "%s: failed to get %s: %d", __func__,
+ name, ret);
+ }
+
+ return ret;
+}
+
+static void ufs_mtk_init_host_caps(struct ufs_hba *hba)
+{
+ struct ufs_mtk_host *host = ufshcd_get_variant(hba);
+ struct ufs_mtk_crypt_cfg *cfg;
+ struct device *dev = hba->dev;
+ struct regulator *reg;
+ u32 volt;
+
+ host->caps = host->cfg->caps;
+
+ if (!ufs_mtk_is_boost_crypt_enabled(hba))
+ return;
+
+ host->crypt = devm_kzalloc(dev, sizeof(*(host->crypt)),
+ GFP_KERNEL);
+ if (!host->crypt)
+ goto disable_caps;
+
+ reg = devm_regulator_get_optional(dev, "dvfsrc-vcore");
+ if (IS_ERR(reg)) {
+ dev_info(dev, "failed to get dvfsrc-vcore: %ld",
+ PTR_ERR(reg));
+ goto disable_caps;
+ }
+
+ if (of_property_read_u32(dev->of_node, "boost-crypt-vcore-min",
+ &volt)) {
+ dev_info(dev, "failed to get boost-crypt-vcore-min");
+ goto disable_caps;
+ }
+
+ cfg = host->crypt;
+ if (ufs_mtk_init_host_clk(hba, "crypt_mux",
+ &cfg->clk_crypt_mux))
+ goto disable_caps;
+
+ if (ufs_mtk_init_host_clk(hba, "crypt_lp",
+ &cfg->clk_crypt_lp))
+ goto disable_caps;
+
+ if (ufs_mtk_init_host_clk(hba, "crypt_perf",
+ &cfg->clk_crypt_perf))
+ goto disable_caps;
+
+ cfg->reg_vcore = reg;
+ cfg->vcore_volt = volt;
+ dev_info(dev, "caps: boost-crypt");
+ return;
+
+disable_caps:
+ host->caps &= ~UFS_MTK_CAP_BOOST_CRYPT_ENGINE;
+}
+
/**
* ufs_mtk_setup_clocks - enables/disable clocks
* @hba: host controller instance
@@ -291,12 +497,14 @@ static int ufs_mtk_setup_clocks(struct ufs_hba *hba, bool on,
}
if (clk_pwr_off) {
+ ufs_mtk_boost_crypt(hba, on);
ufs_mtk_setup_ref_clk(hba, on);
ufs_mtk_mphy_power_on(hba, on);
}
} else if (on && status == POST_CHANGE) {
ufs_mtk_mphy_power_on(hba, on);
ufs_mtk_setup_ref_clk(hba, on);
+ ufs_mtk_boost_crypt(hba, on);
}
return ret;
@@ -314,8 +522,9 @@ static int ufs_mtk_setup_clocks(struct ufs_hba *hba, bool on,
*/
static int ufs_mtk_init(struct ufs_hba *hba)
{
- struct ufs_mtk_host *host;
+ const struct of_device_id *id;
struct device *dev = hba->dev;
+ struct ufs_mtk_host *host;
int err = 0;
host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
@@ -328,10 +537,24 @@ static int ufs_mtk_init(struct ufs_hba *hba)
host->hba = hba;
ufshcd_set_variant(hba, host);
+ /* Get host capability and platform data */
+ id = of_match_device(ufs_mtk_of_match, dev);
+ if (!id) {
+ err = -EINVAL;
+ goto out;
+ }
+
+ if (id->data) {
+ host->cfg = (struct ufs_mtk_host_cfg *)id->data;
+ ufs_mtk_init_host_caps(hba);
+ }
+
err = ufs_mtk_bind_mphy(hba);
if (err)
goto out_variant_clear;
+ ufs_mtk_init_reset(hba);
+
/* Enable runtime autosuspend */
hba->caps |= UFSHCD_CAP_RPM_AUTOSUSPEND;
@@ -416,7 +639,7 @@ static int ufs_mtk_pwr_change_notify(struct ufs_hba *hba,
return ret;
}
-static int ufs_mtk_unipro_set_pm(struct ufs_hba *hba, u32 lpm)
+static int ufs_mtk_unipro_set_pm(struct ufs_hba *hba, bool lpm)
{
int ret;
struct ufs_mtk_host *host = ufshcd_get_variant(hba);
@@ -424,8 +647,14 @@ static int ufs_mtk_unipro_set_pm(struct ufs_hba *hba, u32 lpm)
ret = ufshcd_dme_set(hba,
UIC_ARG_MIB_SEL(VS_UNIPROPOWERDOWNCONTROL, 0),
lpm);
- if (!ret)
+ if (!ret || !lpm) {
+ /*
+ * Forcibly set as non-LPM mode if UIC commands is failed
+ * to use default hba_enable_delay_us value for re-enabling
+ * the host.
+ */
host->unipro_lpm = lpm;
+ }
return ret;
}
@@ -435,7 +664,9 @@ static int ufs_mtk_pre_link(struct ufs_hba *hba)
int ret;
u32 tmp;
- ufs_mtk_unipro_set_pm(hba, 0);
+ ret = ufs_mtk_unipro_set_pm(hba, false);
+ if (ret)
+ return ret;
/*
* Setting PA_Local_TX_LCC_Enable to 0 before link startup
@@ -543,7 +774,7 @@ static int ufs_mtk_link_set_hpm(struct ufs_hba *hba)
if (err)
return err;
- err = ufs_mtk_unipro_set_pm(hba, 0);
+ err = ufs_mtk_unipro_set_pm(hba, false);
if (err)
return err;
@@ -564,10 +795,10 @@ static int ufs_mtk_link_set_lpm(struct ufs_hba *hba)
{
int err;
- err = ufs_mtk_unipro_set_pm(hba, 1);
+ err = ufs_mtk_unipro_set_pm(hba, true);
if (err) {
/* Resume UniPro state for following error recovery */
- ufs_mtk_unipro_set_pm(hba, 0);
+ ufs_mtk_unipro_set_pm(hba, false);
return err;
}
@@ -669,22 +900,16 @@ static int ufs_mtk_apply_dev_quirks(struct ufs_hba *hba)
static void ufs_mtk_fixup_dev_quirks(struct ufs_hba *hba)
{
- struct ufs_dev_info *dev_info = &hba->dev_info;
- u16 mid = dev_info->wmanufacturerid;
-
ufshcd_fixup_dev_quirks(hba, ufs_mtk_dev_fixups);
-
- if (mid == UFS_VENDOR_SAMSUNG)
- hba->dev_quirks &= ~UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE;
}
-/**
+/*
* struct ufs_hba_mtk_vops - UFS MTK specific variant operations
*
* The variant operations configure the necessary controller and PHY
* handshake during initialization.
*/
-static struct ufs_hba_variant_ops ufs_hba_mtk_vops = {
+static const struct ufs_hba_variant_ops ufs_hba_mtk_vops = {
.name = "mediatek.ufshci",
.init = ufs_mtk_init,
.setup_clocks = ufs_mtk_setup_clocks,
@@ -733,11 +958,6 @@ static int ufs_mtk_remove(struct platform_device *pdev)
return 0;
}
-static const struct of_device_id ufs_mtk_of_match[] = {
- { .compatible = "mediatek,mt8183-ufshci"},
- {},
-};
-
static const struct dev_pm_ops ufs_mtk_pm_ops = {
.suspend = ufshcd_pltfrm_suspend,
.resume = ufshcd_pltfrm_resume,
diff --git a/drivers/scsi/ufs/ufs-mediatek.h b/drivers/scsi/ufs/ufs-mediatek.h
index 8ed24d5fcff9..2b6a1312c9bc 100644
--- a/drivers/scsi/ufs/ufs-mediatek.h
+++ b/drivers/scsi/ufs/ufs-mediatek.h
@@ -33,8 +33,8 @@
/*
* Vendor specific pre-defined parameters
*/
-#define UFS_MTK_LIMIT_NUM_LANES_RX 1
-#define UFS_MTK_LIMIT_NUM_LANES_TX 1
+#define UFS_MTK_LIMIT_NUM_LANES_RX 2
+#define UFS_MTK_LIMIT_NUM_LANES_TX 2
#define UFS_MTK_LIMIT_HSGEAR_RX UFS_HS_G3
#define UFS_MTK_LIMIT_HSGEAR_TX UFS_HS_G3
#define UFS_MTK_LIMIT_PWMGEAR_RX UFS_PWM_G4
@@ -89,9 +89,34 @@ enum {
TX_CLK_GATE_EN = 3,
};
+/*
+ * Host capability
+ */
+enum ufs_mtk_host_caps {
+ UFS_MTK_CAP_BOOST_CRYPT_ENGINE = 1 << 0,
+};
+
+struct ufs_mtk_crypt_cfg {
+ struct regulator *reg_vcore;
+ struct clk *clk_crypt_perf;
+ struct clk *clk_crypt_mux;
+ struct clk *clk_crypt_lp;
+ int vcore_volt;
+};
+
+struct ufs_mtk_host_cfg {
+ enum ufs_mtk_host_caps caps;
+};
+
struct ufs_mtk_host {
struct ufs_hba *hba;
struct phy *mphy;
+ struct ufs_mtk_host_cfg *cfg;
+ struct ufs_mtk_crypt_cfg *crypt;
+ enum ufs_mtk_host_caps caps;
+ struct reset_control *hci_reset;
+ struct reset_control *unipro_reset;
+ struct reset_control *crypto_reset;
bool mphy_powered_on;
bool unipro_lpm;
bool ref_clk_enabled;
diff --git a/drivers/scsi/ufs/ufs-qcom.c b/drivers/scsi/ufs/ufs-qcom.c
index d0d75527830e..f9d6ef356540 100644
--- a/drivers/scsi/ufs/ufs-qcom.c
+++ b/drivers/scsi/ufs/ufs-qcom.c
@@ -621,218 +621,6 @@ static int ufs_qcom_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
return 0;
}
-#ifdef CONFIG_MSM_BUS_SCALING
-static int ufs_qcom_get_bus_vote(struct ufs_qcom_host *host,
- const char *speed_mode)
-{
- struct device *dev = host->hba->dev;
- struct device_node *np = dev->of_node;
- int err;
- const char *key = "qcom,bus-vector-names";
-
- if (!speed_mode) {
- err = -EINVAL;
- goto out;
- }
-
- if (host->bus_vote.is_max_bw_needed && !!strcmp(speed_mode, "MIN"))
- err = of_property_match_string(np, key, "MAX");
- else
- err = of_property_match_string(np, key, speed_mode);
-
-out:
- if (err < 0)
- dev_err(dev, "%s: Invalid %s mode %d\n",
- __func__, speed_mode, err);
- return err;
-}
-
-static void ufs_qcom_get_speed_mode(struct ufs_pa_layer_attr *p, char *result)
-{
- int gear = max_t(u32, p->gear_rx, p->gear_tx);
- int lanes = max_t(u32, p->lane_rx, p->lane_tx);
- int pwr;
-
- /* default to PWM Gear 1, Lane 1 if power mode is not initialized */
- if (!gear)
- gear = 1;
-
- if (!lanes)
- lanes = 1;
-
- if (!p->pwr_rx && !p->pwr_tx) {
- pwr = SLOWAUTO_MODE;
- snprintf(result, BUS_VECTOR_NAME_LEN, "MIN");
- } else if (p->pwr_rx == FAST_MODE || p->pwr_rx == FASTAUTO_MODE ||
- p->pwr_tx == FAST_MODE || p->pwr_tx == FASTAUTO_MODE) {
- pwr = FAST_MODE;
- snprintf(result, BUS_VECTOR_NAME_LEN, "%s_R%s_G%d_L%d", "HS",
- p->hs_rate == PA_HS_MODE_B ? "B" : "A", gear, lanes);
- } else {
- pwr = SLOW_MODE;
- snprintf(result, BUS_VECTOR_NAME_LEN, "%s_G%d_L%d",
- "PWM", gear, lanes);
- }
-}
-
-static int __ufs_qcom_set_bus_vote(struct ufs_qcom_host *host, int vote)
-{
- int err = 0;
-
- if (vote != host->bus_vote.curr_vote) {
- err = msm_bus_scale_client_update_request(
- host->bus_vote.client_handle, vote);
- if (err) {
- dev_err(host->hba->dev,
- "%s: msm_bus_scale_client_update_request() failed: bus_client_handle=0x%x, vote=%d, err=%d\n",
- __func__, host->bus_vote.client_handle,
- vote, err);
- goto out;
- }
-
- host->bus_vote.curr_vote = vote;
- }
-out:
- return err;
-}
-
-static int ufs_qcom_update_bus_bw_vote(struct ufs_qcom_host *host)
-{
- int vote;
- int err = 0;
- char mode[BUS_VECTOR_NAME_LEN];
-
- ufs_qcom_get_speed_mode(&host->dev_req_params, mode);
-
- vote = ufs_qcom_get_bus_vote(host, mode);
- if (vote >= 0)
- err = __ufs_qcom_set_bus_vote(host, vote);
- else
- err = vote;
-
- if (err)
- dev_err(host->hba->dev, "%s: failed %d\n", __func__, err);
- else
- host->bus_vote.saved_vote = vote;
- return err;
-}
-
-static int ufs_qcom_set_bus_vote(struct ufs_hba *hba, bool on)
-{
- struct ufs_qcom_host *host = ufshcd_get_variant(hba);
- int vote, err;
-
- /*
- * In case ufs_qcom_init() is not yet done, simply ignore.
- * This ufs_qcom_set_bus_vote() shall be called from
- * ufs_qcom_init() after init is done.
- */
- if (!host)
- return 0;
-
- if (on) {
- vote = host->bus_vote.saved_vote;
- if (vote == host->bus_vote.min_bw_vote)
- ufs_qcom_update_bus_bw_vote(host);
- } else {
- vote = host->bus_vote.min_bw_vote;
- }
-
- err = __ufs_qcom_set_bus_vote(host, vote);
- if (err)
- dev_err(hba->dev, "%s: set bus vote failed %d\n",
- __func__, err);
-
- return err;
-}
-
-static ssize_t
-show_ufs_to_mem_max_bus_bw(struct device *dev, struct device_attribute *attr,
- char *buf)
-{
- struct ufs_hba *hba = dev_get_drvdata(dev);
- struct ufs_qcom_host *host = ufshcd_get_variant(hba);
-
- return snprintf(buf, PAGE_SIZE, "%u\n",
- host->bus_vote.is_max_bw_needed);
-}
-
-static ssize_t
-store_ufs_to_mem_max_bus_bw(struct device *dev, struct device_attribute *attr,
- const char *buf, size_t count)
-{
- struct ufs_hba *hba = dev_get_drvdata(dev);
- struct ufs_qcom_host *host = ufshcd_get_variant(hba);
- uint32_t value;
-
- if (!kstrtou32(buf, 0, &value)) {
- host->bus_vote.is_max_bw_needed = !!value;
- ufs_qcom_update_bus_bw_vote(host);
- }
-
- return count;
-}
-
-static int ufs_qcom_bus_register(struct ufs_qcom_host *host)
-{
- int err;
- struct msm_bus_scale_pdata *bus_pdata;
- struct device *dev = host->hba->dev;
- struct platform_device *pdev = to_platform_device(dev);
- struct device_node *np = dev->of_node;
-
- bus_pdata = msm_bus_cl_get_pdata(pdev);
- if (!bus_pdata) {
- dev_err(dev, "%s: failed to get bus vectors\n", __func__);
- err = -ENODATA;
- goto out;
- }
-
- err = of_property_count_strings(np, "qcom,bus-vector-names");
- if (err < 0 || err != bus_pdata->num_usecases) {
- dev_err(dev, "%s: qcom,bus-vector-names not specified correctly %d\n",
- __func__, err);
- goto out;
- }
-
- host->bus_vote.client_handle = msm_bus_scale_register_client(bus_pdata);
- if (!host->bus_vote.client_handle) {
- dev_err(dev, "%s: msm_bus_scale_register_client failed\n",
- __func__);
- err = -EFAULT;
- goto out;
- }
-
- /* cache the vote index for minimum and maximum bandwidth */
- host->bus_vote.min_bw_vote = ufs_qcom_get_bus_vote(host, "MIN");
- host->bus_vote.max_bw_vote = ufs_qcom_get_bus_vote(host, "MAX");
-
- host->bus_vote.max_bus_bw.show = show_ufs_to_mem_max_bus_bw;
- host->bus_vote.max_bus_bw.store = store_ufs_to_mem_max_bus_bw;
- sysfs_attr_init(&host->bus_vote.max_bus_bw.attr);
- host->bus_vote.max_bus_bw.attr.name = "max_bus_bw";
- host->bus_vote.max_bus_bw.attr.mode = S_IRUGO | S_IWUSR;
- err = device_create_file(dev, &host->bus_vote.max_bus_bw);
-out:
- return err;
-}
-#else /* CONFIG_MSM_BUS_SCALING */
-static int ufs_qcom_update_bus_bw_vote(struct ufs_qcom_host *host)
-{
- return 0;
-}
-
-static int ufs_qcom_set_bus_vote(struct ufs_hba *host, bool on)
-{
- return 0;
-}
-
-static int ufs_qcom_bus_register(struct ufs_qcom_host *host)
-{
- return 0;
-}
-#endif /* CONFIG_MSM_BUS_SCALING */
-
static void ufs_qcom_dev_ref_clk_ctrl(struct ufs_qcom_host *host, bool enable)
{
if (host->dev_ref_clk_ctrl_mmio &&
@@ -976,7 +764,6 @@ static int ufs_qcom_pwr_change_notify(struct ufs_hba *hba,
/* cache the power mode parameters to use internally */
memcpy(&host->dev_req_params,
dev_req_params, sizeof(*dev_req_params));
- ufs_qcom_update_bus_bw_vote(host);
/* disable the device ref clock if entered PWM mode */
if (ufshcd_is_hs_mode(&hba->pwr_info) &&
@@ -1107,9 +894,7 @@ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on,
switch (status) {
case PRE_CHANGE:
- if (on) {
- err = ufs_qcom_set_bus_vote(hba, true);
- } else {
+ if (!on) {
if (!ufs_qcom_is_link_active(hba)) {
/* disable device ref_clk */
ufs_qcom_dev_ref_clk_ctrl(host, false);
@@ -1121,8 +906,6 @@ static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on,
/* enable the device ref clock for HS mode*/
if (ufshcd_is_hs_mode(&hba->pwr_info))
ufs_qcom_dev_ref_clk_ctrl(host, true);
- } else {
- err = ufs_qcom_set_bus_vote(hba, false);
}
break;
}
@@ -1264,10 +1047,6 @@ static int ufs_qcom_init(struct ufs_hba *hba)
goto out_variant_clear;
}
- err = ufs_qcom_bus_register(host);
- if (err)
- goto out_variant_clear;
-
ufs_qcom_get_controller_revision(hba, &host->hw_ver.major,
&host->hw_ver.minor, &host->hw_ver.step);
@@ -1307,7 +1086,6 @@ static int ufs_qcom_init(struct ufs_hba *hba)
if (err)
goto out_variant_clear;
- ufs_qcom_set_bus_vote(hba, true);
ufs_qcom_setup_clocks(hba, true, POST_CHANGE);
if (hba->dev->id < MAX_UFS_QCOM_HOSTS)
@@ -1446,7 +1224,6 @@ static int ufs_qcom_clk_scale_notify(struct ufs_hba *hba,
dev_req_params->pwr_rx,
dev_req_params->hs_rate,
false);
- ufs_qcom_update_bus_bw_vote(host);
}
out:
@@ -1614,9 +1391,6 @@ int ufs_qcom_testbus_config(struct ufs_qcom_host *host)
*/
}
mask <<= offset;
-
- pm_runtime_get_sync(host->hba->dev);
- ufshcd_hold(host->hba, false);
ufshcd_rmwl(host->hba, TEST_BUS_SEL,
(u32)host->testbus.select_major << 19,
REG_UFS_CFG1);
@@ -1629,50 +1403,16 @@ int ufs_qcom_testbus_config(struct ufs_qcom_host *host)
* committed before returning.
*/
mb();
- ufshcd_release(host->hba);
- pm_runtime_put_sync(host->hba->dev);
return 0;
}
-static void ufs_qcom_testbus_read(struct ufs_hba *hba)
-{
- ufshcd_dump_regs(hba, UFS_TEST_BUS, 4, "UFS_TEST_BUS ");
-}
-
-static void ufs_qcom_print_unipro_testbus(struct ufs_hba *hba)
-{
- struct ufs_qcom_host *host = ufshcd_get_variant(hba);
- u32 *testbus = NULL;
- int i, nminor = 256, testbus_len = nminor * sizeof(u32);
-
- testbus = kmalloc(testbus_len, GFP_KERNEL);
- if (!testbus)
- return;
-
- host->testbus.select_major = TSTBUS_UNIPRO;
- for (i = 0; i < nminor; i++) {
- host->testbus.select_minor = i;
- ufs_qcom_testbus_config(host);
- testbus[i] = ufshcd_readl(hba, UFS_TEST_BUS);
- }
- print_hex_dump(KERN_ERR, "UNIPRO_TEST_BUS ", DUMP_PREFIX_OFFSET,
- 16, 4, testbus, testbus_len, false);
- kfree(testbus);
-}
-
static void ufs_qcom_dump_dbg_regs(struct ufs_hba *hba)
{
ufshcd_dump_regs(hba, REG_UFS_SYS1CLK_1US, 16 * 4,
"HCI Vendor Specific Registers ");
- /* sleep a bit intermittently as we are dumping too much data */
ufs_qcom_print_hw_debug_reg_all(hba, NULL, ufs_qcom_dump_regs_wrapper);
- udelay(1000);
- ufs_qcom_testbus_read(hba);
- udelay(1000);
- ufs_qcom_print_unipro_testbus(hba);
- udelay(1000);
}
/**
diff --git a/drivers/scsi/ufs/ufs-qcom.h b/drivers/scsi/ufs/ufs-qcom.h
index 97247d17e258..3f4922743b3e 100644
--- a/drivers/scsi/ufs/ufs-qcom.h
+++ b/drivers/scsi/ufs/ufs-qcom.h
@@ -174,16 +174,6 @@ static inline void ufs_qcom_deassert_reset(struct ufs_hba *hba)
mb();
}
-struct ufs_qcom_bus_vote {
- uint32_t client_handle;
- uint32_t curr_vote;
- int min_bw_vote;
- int max_bw_vote;
- int saved_vote;
- bool is_max_bw_needed;
- struct device_attribute max_bus_bw;
-};
-
/* Host controller hardware version: major.minor.step */
struct ufs_hw_version {
u16 step;
@@ -216,7 +206,6 @@ struct ufs_qcom_host {
struct phy *generic_phy;
struct ufs_hba *hba;
- struct ufs_qcom_bus_vote bus_vote;
struct ufs_pa_layer_attr dev_req_params;
struct clk *rx_l0_sync_clk;
struct clk *tx_l0_sync_clk;
diff --git a/drivers/scsi/ufs/ufs-sysfs.c b/drivers/scsi/ufs/ufs-sysfs.c
index 2d71d232a69d..bdcd27faa054 100644
--- a/drivers/scsi/ufs/ufs-sysfs.c
+++ b/drivers/scsi/ufs/ufs-sysfs.c
@@ -16,6 +16,7 @@ static const char *ufschd_uic_link_state_to_string(
case UIC_LINK_OFF_STATE: return "OFF";
case UIC_LINK_ACTIVE_STATE: return "ACTIVE";
case UIC_LINK_HIBERN8_STATE: return "HIBERN8";
+ case UIC_LINK_BROKEN_STATE: return "BROKEN";
default: return "UNKNOWN";
}
}
@@ -145,12 +146,19 @@ static u32 ufshcd_us_to_ahit(unsigned int timer)
static ssize_t auto_hibern8_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
+ u32 ahit;
struct ufs_hba *hba = dev_get_drvdata(dev);
if (!ufshcd_is_auto_hibern8_supported(hba))
return -EOPNOTSUPP;
- return snprintf(buf, PAGE_SIZE, "%d\n", ufshcd_ahit_to_us(hba->ahit));
+ pm_runtime_get_sync(hba->dev);
+ ufshcd_hold(hba, false);
+ ahit = ufshcd_readl(hba, REG_AUTO_HIBERNATE_IDLE_TIMER);
+ ufshcd_release(hba);
+ pm_runtime_put_sync(hba->dev);
+
+ return scnprintf(buf, PAGE_SIZE, "%d\n", ufshcd_ahit_to_us(ahit));
}
static ssize_t auto_hibern8_store(struct device *dev,
diff --git a/drivers/scsi/ufs/ufshcd-crypto.c b/drivers/scsi/ufs/ufshcd-crypto.c
index d2edbd960ebf..07310b12a5dc 100644
--- a/drivers/scsi/ufs/ufshcd-crypto.c
+++ b/drivers/scsi/ufs/ufshcd-crypto.c
@@ -59,7 +59,7 @@ static int ufshcd_crypto_keyslot_program(struct blk_keyslot_manager *ksm,
u8 data_unit_mask = key->crypto_cfg.data_unit_size / 512;
int i;
int cap_idx = -1;
- union ufs_crypto_cfg_entry cfg = { 0 };
+ union ufs_crypto_cfg_entry cfg = {};
int err;
BUILD_BUG_ON(UFS_CRYPTO_KEY_SIZE_INVALID != 0);
@@ -100,7 +100,7 @@ static int ufshcd_clear_keyslot(struct ufs_hba *hba, int slot)
* Clear the crypto cfg on the device. Clearing CFGE
* might not be sufficient, so just clear the entire cfg.
*/
- union ufs_crypto_cfg_entry cfg = { 0 };
+ union ufs_crypto_cfg_entry cfg = {};
return ufshcd_program_key(hba, &cfg, slot);
}
diff --git a/drivers/scsi/ufs/ufshcd-pci.c b/drivers/scsi/ufs/ufshcd-pci.c
index 5a95a7bfbab0..df3a564c3e33 100644
--- a/drivers/scsi/ufs/ufshcd-pci.c
+++ b/drivers/scsi/ufs/ufshcd-pci.c
@@ -13,6 +13,14 @@
#include "ufshcd.h"
#include <linux/pci.h>
#include <linux/pm_runtime.h>
+#include <linux/pm_qos.h>
+#include <linux/debugfs.h>
+
+struct intel_host {
+ u32 active_ltr;
+ u32 idle_ltr;
+ struct dentry *debugfs_root;
+};
static int ufs_intel_disable_lcc(struct ufs_hba *hba)
{
@@ -44,20 +52,134 @@ static int ufs_intel_link_startup_notify(struct ufs_hba *hba,
return err;
}
+#define INTEL_ACTIVELTR 0x804
+#define INTEL_IDLELTR 0x808
+
+#define INTEL_LTR_REQ BIT(15)
+#define INTEL_LTR_SCALE_MASK GENMASK(11, 10)
+#define INTEL_LTR_SCALE_1US (2 << 10)
+#define INTEL_LTR_SCALE_32US (3 << 10)
+#define INTEL_LTR_VALUE_MASK GENMASK(9, 0)
+
+static void intel_cache_ltr(struct ufs_hba *hba)
+{
+ struct intel_host *host = ufshcd_get_variant(hba);
+
+ host->active_ltr = readl(hba->mmio_base + INTEL_ACTIVELTR);
+ host->idle_ltr = readl(hba->mmio_base + INTEL_IDLELTR);
+}
+
+static void intel_ltr_set(struct device *dev, s32 val)
+{
+ struct ufs_hba *hba = dev_get_drvdata(dev);
+ struct intel_host *host = ufshcd_get_variant(hba);
+ u32 ltr;
+
+ pm_runtime_get_sync(dev);
+
+ /*
+ * Program latency tolerance (LTR) accordingly what has been asked
+ * by the PM QoS layer or disable it in case we were passed
+ * negative value or PM_QOS_LATENCY_ANY.
+ */
+ ltr = readl(hba->mmio_base + INTEL_ACTIVELTR);
+
+ if (val == PM_QOS_LATENCY_ANY || val < 0) {
+ ltr &= ~INTEL_LTR_REQ;
+ } else {
+ ltr |= INTEL_LTR_REQ;
+ ltr &= ~INTEL_LTR_SCALE_MASK;
+ ltr &= ~INTEL_LTR_VALUE_MASK;
+
+ if (val > INTEL_LTR_VALUE_MASK) {
+ val >>= 5;
+ if (val > INTEL_LTR_VALUE_MASK)
+ val = INTEL_LTR_VALUE_MASK;
+ ltr |= INTEL_LTR_SCALE_32US | val;
+ } else {
+ ltr |= INTEL_LTR_SCALE_1US | val;
+ }
+ }
+
+ if (ltr == host->active_ltr)
+ goto out;
+
+ writel(ltr, hba->mmio_base + INTEL_ACTIVELTR);
+ writel(ltr, hba->mmio_base + INTEL_IDLELTR);
+
+ /* Cache the values into intel_host structure */
+ intel_cache_ltr(hba);
+out:
+ pm_runtime_put(dev);
+}
+
+static void intel_ltr_expose(struct device *dev)
+{
+ dev->power.set_latency_tolerance = intel_ltr_set;
+ dev_pm_qos_expose_latency_tolerance(dev);
+}
+
+static void intel_ltr_hide(struct device *dev)
+{
+ dev_pm_qos_hide_latency_tolerance(dev);
+ dev->power.set_latency_tolerance = NULL;
+}
+
+static void intel_add_debugfs(struct ufs_hba *hba)
+{
+ struct dentry *dir = debugfs_create_dir(dev_name(hba->dev), NULL);
+ struct intel_host *host = ufshcd_get_variant(hba);
+
+ intel_cache_ltr(hba);
+
+ host->debugfs_root = dir;
+ debugfs_create_x32("active_ltr", 0444, dir, &host->active_ltr);
+ debugfs_create_x32("idle_ltr", 0444, dir, &host->idle_ltr);
+}
+
+static void intel_remove_debugfs(struct ufs_hba *hba)
+{
+ struct intel_host *host = ufshcd_get_variant(hba);
+
+ debugfs_remove_recursive(host->debugfs_root);
+}
+
+static int ufs_intel_common_init(struct ufs_hba *hba)
+{
+ struct intel_host *host;
+
+ host = devm_kzalloc(hba->dev, sizeof(*host), GFP_KERNEL);
+ if (!host)
+ return -ENOMEM;
+ ufshcd_set_variant(hba, host);
+ intel_ltr_expose(hba->dev);
+ intel_add_debugfs(hba);
+ return 0;
+}
+
+static void ufs_intel_common_exit(struct ufs_hba *hba)
+{
+ intel_remove_debugfs(hba);
+ intel_ltr_hide(hba->dev);
+}
+
static int ufs_intel_ehl_init(struct ufs_hba *hba)
{
hba->quirks |= UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8;
- return 0;
+ return ufs_intel_common_init(hba);
}
static struct ufs_hba_variant_ops ufs_intel_cnl_hba_vops = {
.name = "intel-pci",
+ .init = ufs_intel_common_init,
+ .exit = ufs_intel_common_exit,
.link_startup_notify = ufs_intel_link_startup_notify,
};
static struct ufs_hba_variant_ops ufs_intel_ehl_hba_vops = {
.name = "intel-pci",
.init = ufs_intel_ehl_init,
+ .exit = ufs_intel_common_exit,
.link_startup_notify = ufs_intel_link_startup_notify,
};
@@ -162,6 +284,8 @@ ufshcd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
return err;
}
+ pci_set_drvdata(pdev, hba);
+
hba->vops = (struct ufs_hba_variant_ops *)id->driver_data;
err = ufshcd_init(hba, mmio_base, pdev->irq);
@@ -171,7 +295,6 @@ ufshcd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
return err;
}
- pci_set_drvdata(pdev, hba);
pm_runtime_put_noidle(&pdev->dev);
pm_runtime_allow(&pdev->dev);
diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
index 1d157ff58d81..b8f573a02713 100644
--- a/drivers/scsi/ufs/ufshcd.c
+++ b/drivers/scsi/ufs/ufshcd.c
@@ -15,6 +15,7 @@
#include <linux/of.h>
#include <linux/bitfield.h>
#include <linux/blk-pm.h>
+#include <linux/blkdev.h>
#include "ufshcd.h"
#include "ufs_quirks.h"
#include "unipro.h"
@@ -35,8 +36,8 @@
/* NOP OUT retries waiting for NOP IN response */
#define NOP_OUT_RETRIES 10
-/* Timeout after 30 msecs if NOP OUT hangs without response */
-#define NOP_OUT_TIMEOUT 30 /* msecs */
+/* Timeout after 50 msecs if NOP OUT hangs without response */
+#define NOP_OUT_TIMEOUT 50 /* msecs */
/* Query request retries */
#define QUERY_REQ_RETRIES 3
@@ -73,6 +74,9 @@
/* Default value of wait time before gating device ref clock */
#define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
+/* Polling time to wait for fDeviceInit */
+#define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
+
#define ufshcd_toggle_vreg(_dev, _vreg, _on) \
({ \
int _ret; \
@@ -125,7 +129,8 @@ enum {
UFSHCD_STATE_RESET,
UFSHCD_STATE_ERROR,
UFSHCD_STATE_OPERATIONAL,
- UFSHCD_STATE_EH_SCHEDULED,
+ UFSHCD_STATE_EH_SCHEDULED_FATAL,
+ UFSHCD_STATE_EH_SCHEDULED_NON_FATAL,
};
/* UFSHCD error handling flags */
@@ -141,6 +146,7 @@ enum {
UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
+ UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
};
#define ufshcd_set_eh_in_progress(h) \
@@ -228,6 +234,12 @@ static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
static irqreturn_t ufshcd_intr(int irq, void *__hba);
static int ufshcd_change_power_mode(struct ufs_hba *hba,
struct ufs_pa_layer_attr *pwr_mode);
+static void ufshcd_schedule_eh_work(struct ufs_hba *hba);
+static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
+static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
+static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
+ struct ufs_vreg *vreg);
+static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
static int ufshcd_wb_buf_flush_enable(struct ufs_hba *hba);
static int ufshcd_wb_buf_flush_disable(struct ufs_hba *hba);
static int ufshcd_wb_ctrl(struct ufs_hba *hba, bool enable);
@@ -411,15 +423,6 @@ static void ufshcd_print_err_hist(struct ufs_hba *hba,
static void ufshcd_print_host_regs(struct ufs_hba *hba)
{
ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
- dev_err(hba->dev, "hba->ufs_version = 0x%x, hba->capabilities = 0x%x\n",
- hba->ufs_version, hba->capabilities);
- dev_err(hba->dev,
- "hba->outstanding_reqs = 0x%x, hba->outstanding_tasks = 0x%x\n",
- (u32)hba->outstanding_reqs, (u32)hba->outstanding_tasks);
- dev_err(hba->dev,
- "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt = %d\n",
- ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
- hba->ufs_stats.hibern8_exit_cnt);
ufshcd_print_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err");
ufshcd_print_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err");
@@ -438,8 +441,6 @@ static void ufshcd_print_host_regs(struct ufs_hba *hba)
ufshcd_print_err_hist(hba, &hba->ufs_stats.host_reset, "host_reset");
ufshcd_print_err_hist(hba, &hba->ufs_stats.task_abort, "task_abort");
- ufshcd_print_clk_freqs(hba);
-
ufshcd_vops_dbg_register_dump(hba);
}
@@ -474,6 +475,9 @@ void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
prdt_length = le16_to_cpu(
lrbp->utr_descriptor_ptr->prd_table_length);
+ if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
+ prdt_length /= sizeof(struct ufshcd_sg_entry);
+
dev_err(hba->dev,
"UPIU[%d] - PRDT - %d entries phys@0x%llx\n",
tag, prdt_length,
@@ -499,6 +503,8 @@ static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
static void ufshcd_print_host_state(struct ufs_hba *hba)
{
+ struct scsi_device *sdev_ufs = hba->sdev_ufs_device;
+
dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
hba->outstanding_reqs, hba->outstanding_tasks);
@@ -511,12 +517,24 @@ static void ufshcd_print_host_state(struct ufs_hba *hba)
dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
hba->auto_bkops_enabled, hba->host->host_self_blocked);
dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
+ dev_err(hba->dev,
+ "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
+ ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
+ hba->ufs_stats.hibern8_exit_cnt);
+ dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
+ ktime_to_us(hba->ufs_stats.last_intr_ts),
+ hba->ufs_stats.last_intr_status);
dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
hba->eh_flags, hba->req_abort_count);
- dev_err(hba->dev, "Host capabilities=0x%x, caps=0x%x\n",
- hba->capabilities, hba->caps);
+ dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
+ hba->ufs_version, hba->capabilities, hba->caps);
dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
hba->dev_quirks);
+ if (sdev_ufs)
+ dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
+ sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
+
+ ufshcd_print_clk_freqs(hba);
}
/**
@@ -1569,11 +1587,6 @@ int ufshcd_hold(struct ufs_hba *hba, bool async)
spin_lock_irqsave(hba->host->host_lock, flags);
hba->clk_gating.active_reqs++;
- if (ufshcd_eh_in_progress(hba)) {
- spin_unlock_irqrestore(hba->host->host_lock, flags);
- return 0;
- }
-
start:
switch (hba->clk_gating.state) {
case CLKS_ON:
@@ -1653,6 +1666,7 @@ static void ufshcd_gate_work(struct work_struct *work)
struct ufs_hba *hba = container_of(work, struct ufs_hba,
clk_gating.gate_work.work);
unsigned long flags;
+ int ret;
spin_lock_irqsave(hba->host->host_lock, flags);
/*
@@ -1679,8 +1693,11 @@ static void ufshcd_gate_work(struct work_struct *work)
/* put the link into hibern8 mode before turning off clocks */
if (ufshcd_can_hibern8_during_gating(hba)) {
- if (ufshcd_uic_hibern8_enter(hba)) {
+ ret = ufshcd_uic_hibern8_enter(hba);
+ if (ret) {
hba->clk_gating.state = CLKS_ON;
+ dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
+ __func__, ret);
trace_ufshcd_clk_gating(dev_name(hba->dev),
hba->clk_gating.state);
goto out;
@@ -1725,11 +1742,10 @@ static void __ufshcd_release(struct ufs_hba *hba)
hba->clk_gating.active_reqs--;
- if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended
- || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
- || ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks
- || hba->active_uic_cmd || hba->uic_async_done
- || ufshcd_eh_in_progress(hba))
+ if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
+ hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
+ ufshcd_any_tag_in_use(hba) || hba->outstanding_tasks ||
+ hba->active_uic_cmd || hba->uic_async_done)
return;
hba->clk_gating.state = REQ_CLKS_OFF;
@@ -1842,6 +1858,8 @@ static void ufshcd_init_clk_gating(struct ufs_hba *hba)
if (!ufshcd_is_clkgating_allowed(hba))
return;
+ hba->clk_gating.state = CLKS_ON;
+
hba->clk_gating.delay_ms = 150;
INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
@@ -2394,12 +2412,13 @@ static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
}
/**
- * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
+ * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
* for Device Management Purposes
* @hba: per adapter instance
* @lrbp: pointer to local reference block
*/
-static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
+static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
+ struct ufshcd_lrb *lrbp)
{
u8 upiu_flags;
int ret = 0;
@@ -2509,34 +2528,6 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
if (!down_read_trylock(&hba->clk_scaling_lock))
return SCSI_MLQUEUE_HOST_BUSY;
- spin_lock_irqsave(hba->host->host_lock, flags);
- switch (hba->ufshcd_state) {
- case UFSHCD_STATE_OPERATIONAL:
- break;
- case UFSHCD_STATE_EH_SCHEDULED:
- case UFSHCD_STATE_RESET:
- err = SCSI_MLQUEUE_HOST_BUSY;
- goto out_unlock;
- case UFSHCD_STATE_ERROR:
- set_host_byte(cmd, DID_ERROR);
- cmd->scsi_done(cmd);
- goto out_unlock;
- default:
- dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
- __func__, hba->ufshcd_state);
- set_host_byte(cmd, DID_BAD_TARGET);
- cmd->scsi_done(cmd);
- goto out_unlock;
- }
-
- /* if error handling is in progress, don't issue commands */
- if (ufshcd_eh_in_progress(hba)) {
- set_host_byte(cmd, DID_ERROR);
- cmd->scsi_done(cmd);
- goto out_unlock;
- }
- spin_unlock_irqrestore(hba->host->host_lock, flags);
-
hba->req_abort_count = 0;
err = ufshcd_hold(hba, true);
@@ -2544,7 +2535,8 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
err = SCSI_MLQUEUE_HOST_BUSY;
goto out;
}
- WARN_ON(hba->clk_gating.state != CLKS_ON);
+ WARN_ON(ufshcd_is_clkgating_allowed(hba) &&
+ (hba->clk_gating.state != CLKS_ON));
lrbp = &hba->lrb[tag];
@@ -2571,11 +2563,51 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
/* Make sure descriptors are ready before ringing the doorbell */
wmb();
- /* issue command to the controller */
spin_lock_irqsave(hba->host->host_lock, flags);
+ switch (hba->ufshcd_state) {
+ case UFSHCD_STATE_OPERATIONAL:
+ case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
+ break;
+ case UFSHCD_STATE_EH_SCHEDULED_FATAL:
+ /*
+ * pm_runtime_get_sync() is used at error handling preparation
+ * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
+ * PM ops, it can never be finished if we let SCSI layer keep
+ * retrying it, which gets err handler stuck forever. Neither
+ * can we let the scsi cmd pass through, because UFS is in bad
+ * state, the scsi cmd may eventually time out, which will get
+ * err handler blocked for too long. So, just fail the scsi cmd
+ * sent from PM ops, err handler can recover PM error anyways.
+ */
+ if (hba->pm_op_in_progress) {
+ hba->force_reset = true;
+ set_host_byte(cmd, DID_BAD_TARGET);
+ goto out_compl_cmd;
+ }
+ fallthrough;
+ case UFSHCD_STATE_RESET:
+ err = SCSI_MLQUEUE_HOST_BUSY;
+ goto out_compl_cmd;
+ case UFSHCD_STATE_ERROR:
+ set_host_byte(cmd, DID_ERROR);
+ goto out_compl_cmd;
+ default:
+ dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
+ __func__, hba->ufshcd_state);
+ set_host_byte(cmd, DID_BAD_TARGET);
+ goto out_compl_cmd;
+ }
ufshcd_send_command(hba, tag);
-out_unlock:
spin_unlock_irqrestore(hba->host->host_lock, flags);
+ goto out;
+
+out_compl_cmd:
+ scsi_dma_unmap(lrbp->cmd);
+ lrbp->cmd = NULL;
+ spin_unlock_irqrestore(hba->host->host_lock, flags);
+ ufshcd_release(hba);
+ if (!err)
+ cmd->scsi_done(cmd);
out:
up_read(&hba->clk_scaling_lock);
return err;
@@ -2593,7 +2625,7 @@ static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
ufshcd_prepare_lrbp_crypto(NULL, lrbp);
hba->dev_cmd.type = cmd_type;
- return ufshcd_comp_devman_upiu(hba, lrbp);
+ return ufshcd_compose_devman_upiu(hba, lrbp);
}
static int
@@ -3747,6 +3779,10 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
ufshcd_add_delay_before_dme_cmd(hba);
spin_lock_irqsave(hba->host->host_lock, flags);
+ if (ufshcd_is_link_broken(hba)) {
+ ret = -ENOLINK;
+ goto out_unlock;
+ }
hba->uic_async_done = &uic_async_done;
if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
@@ -3794,6 +3830,11 @@ out:
hba->uic_async_done = NULL;
if (reenable_intr)
ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
+ if (ret) {
+ ufshcd_set_link_broken(hba);
+ ufshcd_schedule_eh_work(hba);
+ }
+out_unlock:
spin_unlock_irqrestore(hba->host->host_lock, flags);
mutex_unlock(&hba->uic_cmd_mutex);
@@ -3863,7 +3904,7 @@ int ufshcd_link_recovery(struct ufs_hba *hba)
}
EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
-static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
+static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
{
int ret;
struct uic_command uic_cmd = {0};
@@ -3876,45 +3917,16 @@ static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
ktime_to_us(ktime_sub(ktime_get(), start)), ret);
- if (ret) {
- int err;
-
+ if (ret)
dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
__func__, ret);
-
- /*
- * If link recovery fails then return error code returned from
- * ufshcd_link_recovery().
- * If link recovery succeeds then return -EAGAIN to attempt
- * hibern8 enter retry again.
- */
- err = ufshcd_link_recovery(hba);
- if (err) {
- dev_err(hba->dev, "%s: link recovery failed", __func__);
- ret = err;
- } else {
- ret = -EAGAIN;
- }
- } else
+ else
ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
POST_CHANGE);
return ret;
}
-static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
-{
- int ret = 0, retries;
-
- for (retries = UIC_HIBERN8_ENTER_RETRIES; retries > 0; retries--) {
- ret = __ufshcd_uic_hibern8_enter(hba);
- if (!ret)
- goto out;
- }
-out:
- return ret;
-}
-
int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
{
struct uic_command uic_cmd = {0};
@@ -3931,7 +3943,6 @@ int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
if (ret) {
dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
__func__, ret);
- ret = ufshcd_link_recovery(hba);
} else {
ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
POST_CHANGE);
@@ -3972,7 +3983,7 @@ void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
{
unsigned long flags;
- if (!ufshcd_is_auto_hibern8_supported(hba) || !hba->ahit)
+ if (!ufshcd_is_auto_hibern8_supported(hba))
return;
spin_lock_irqsave(hba->host->host_lock, flags);
@@ -4065,7 +4076,8 @@ static int ufshcd_change_power_mode(struct ufs_hba *hba,
int ret;
/* if already configured to the requested pwr_mode */
- if (pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
+ if (!hba->force_pmc &&
+ pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
@@ -4175,9 +4187,9 @@ EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
*/
static int ufshcd_complete_dev_init(struct ufs_hba *hba)
{
- int i;
int err;
bool flag_res = true;
+ ktime_t timeout;
err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
@@ -4188,20 +4200,26 @@ static int ufshcd_complete_dev_init(struct ufs_hba *hba)
goto out;
}
- /* poll for max. 1000 iterations for fDeviceInit flag to clear */
- for (i = 0; i < 1000 && !err && flag_res; i++)
- err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
- QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
+ /* Poll fDeviceInit flag to be cleared */
+ timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
+ do {
+ err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
+ QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
+ if (!flag_res)
+ break;
+ usleep_range(5000, 10000);
+ } while (ktime_before(ktime_get(), timeout));
- if (err)
+ if (err) {
dev_err(hba->dev,
- "%s reading fDeviceInit flag failed with error %d\n",
- __func__, err);
- else if (flag_res)
+ "%s reading fDeviceInit flag failed with error %d\n",
+ __func__, err);
+ } else if (flag_res) {
dev_err(hba->dev,
- "%s fDeviceInit was not cleared by the device\n",
- __func__);
-
+ "%s fDeviceInit was not cleared by the device\n",
+ __func__);
+ err = -EBUSY;
+ }
out:
return err;
}
@@ -4258,10 +4276,8 @@ int ufshcd_make_hba_operational(struct ufs_hba *hba)
dev_err(hba->dev,
"Host controller not ready to process requests");
err = -EIO;
- goto out;
}
-out:
return err;
}
EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
@@ -4495,6 +4511,8 @@ link_startup:
if (ret)
goto out;
+ /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
+ ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
ret = ufshcd_make_hba_operational(hba);
out:
if (ret) {
@@ -5299,6 +5317,9 @@ static int ufshcd_wb_toggle_flush_during_h8(struct ufs_hba *hba, bool set)
static inline void ufshcd_wb_toggle_flush(struct ufs_hba *hba, bool enable)
{
+ if (hba->quirks & UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL)
+ return;
+
if (enable)
ufshcd_wb_buf_flush_enable(hba);
else
@@ -5544,16 +5565,129 @@ static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
hba->saved_err &= ~UIC_ERROR;
/* clear NAC error */
hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
- if (!hba->saved_uic_err) {
+ if (!hba->saved_uic_err)
err_handling = false;
- goto out;
- }
}
out:
spin_unlock_irqrestore(hba->host->host_lock, flags);
return err_handling;
}
+/* host lock must be held before calling this func */
+static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
+{
+ return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
+ (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
+}
+
+/* host lock must be held before calling this func */
+static inline void ufshcd_schedule_eh_work(struct ufs_hba *hba)
+{
+ /* handle fatal errors only when link is not in error state */
+ if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
+ if (hba->force_reset || ufshcd_is_link_broken(hba) ||
+ ufshcd_is_saved_err_fatal(hba))
+ hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
+ else
+ hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
+ queue_work(hba->eh_wq, &hba->eh_work);
+ }
+}
+
+static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
+{
+ pm_runtime_get_sync(hba->dev);
+ if (pm_runtime_suspended(hba->dev)) {
+ /*
+ * Don't assume anything of pm_runtime_get_sync(), if
+ * resume fails, irq and clocks can be OFF, and powers
+ * can be OFF or in LPM.
+ */
+ ufshcd_setup_hba_vreg(hba, true);
+ ufshcd_enable_irq(hba);
+ ufshcd_setup_vreg(hba, true);
+ ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
+ ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
+ ufshcd_hold(hba, false);
+ if (!ufshcd_is_clkgating_allowed(hba))
+ ufshcd_setup_clocks(hba, true);
+ ufshcd_release(hba);
+ ufshcd_vops_resume(hba, UFS_RUNTIME_PM);
+ } else {
+ ufshcd_hold(hba, false);
+ if (hba->clk_scaling.is_allowed) {
+ cancel_work_sync(&hba->clk_scaling.suspend_work);
+ cancel_work_sync(&hba->clk_scaling.resume_work);
+ ufshcd_suspend_clkscaling(hba);
+ }
+ }
+}
+
+static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
+{
+ ufshcd_release(hba);
+ if (hba->clk_scaling.is_allowed)
+ ufshcd_resume_clkscaling(hba);
+ pm_runtime_put(hba->dev);
+}
+
+static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
+{
+ return (hba->ufshcd_state == UFSHCD_STATE_ERROR ||
+ (!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
+ ufshcd_is_link_broken(hba))));
+}
+
+#ifdef CONFIG_PM
+static void ufshcd_recover_pm_error(struct ufs_hba *hba)
+{
+ struct Scsi_Host *shost = hba->host;
+ struct scsi_device *sdev;
+ struct request_queue *q;
+ int ret;
+
+ /*
+ * Set RPM status of hba device to RPM_ACTIVE,
+ * this also clears its runtime error.
+ */
+ ret = pm_runtime_set_active(hba->dev);
+ /*
+ * If hba device had runtime error, we also need to resume those
+ * scsi devices under hba in case any of them has failed to be
+ * resumed due to hba runtime resume failure. This is to unblock
+ * blk_queue_enter in case there are bios waiting inside it.
+ */
+ if (!ret) {
+ shost_for_each_device(sdev, shost) {
+ q = sdev->request_queue;
+ if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
+ q->rpm_status == RPM_SUSPENDING))
+ pm_request_resume(q->dev);
+ }
+ }
+}
+#else
+static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
+{
+}
+#endif
+
+static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
+{
+ struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
+ u32 mode;
+
+ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
+
+ if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
+ return true;
+
+ if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
+ return true;
+
+ return false;
+}
+
/**
* ufshcd_err_handler - handle UFS errors that require s/w attention
* @work: pointer to work structure
@@ -5562,23 +5696,36 @@ static void ufshcd_err_handler(struct work_struct *work)
{
struct ufs_hba *hba;
unsigned long flags;
- u32 err_xfer = 0;
- u32 err_tm = 0;
- int err = 0;
+ bool err_xfer = false;
+ bool err_tm = false;
+ int err = 0, pmc_err;
int tag;
- bool needs_reset = false;
+ bool needs_reset = false, needs_restore = false;
hba = container_of(work, struct ufs_hba, eh_work);
- pm_runtime_get_sync(hba->dev);
- ufshcd_hold(hba, false);
-
spin_lock_irqsave(hba->host->host_lock, flags);
- if (hba->ufshcd_state == UFSHCD_STATE_RESET)
+ if (ufshcd_err_handling_should_stop(hba)) {
+ if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
+ hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
+ spin_unlock_irqrestore(hba->host->host_lock, flags);
+ return;
+ }
+ ufshcd_set_eh_in_progress(hba);
+ spin_unlock_irqrestore(hba->host->host_lock, flags);
+ ufshcd_err_handling_prepare(hba);
+ spin_lock_irqsave(hba->host->host_lock, flags);
+ ufshcd_scsi_block_requests(hba);
+ /*
+ * A full reset and restore might have happened after preparation
+ * is finished, double check whether we should stop.
+ */
+ if (ufshcd_err_handling_should_stop(hba)) {
+ if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
+ hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
goto out;
-
+ }
hba->ufshcd_state = UFSHCD_STATE_RESET;
- ufshcd_set_eh_in_progress(hba);
/* Complete requests that have door-bell cleared by h/w */
ufshcd_complete_requests(hba);
@@ -5590,30 +5737,61 @@ static void ufshcd_err_handler(struct work_struct *work)
/* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
ret = ufshcd_quirk_dl_nac_errors(hba);
spin_lock_irqsave(hba->host->host_lock, flags);
- if (!ret)
+ if (!ret && !hba->force_reset && ufshcd_is_link_active(hba))
goto skip_err_handling;
}
- if ((hba->saved_err & INT_FATAL_ERRORS) ||
- (hba->saved_err & UFSHCD_UIC_HIBERN8_MASK) ||
+
+ if (hba->force_reset || ufshcd_is_link_broken(hba) ||
+ ufshcd_is_saved_err_fatal(hba) ||
((hba->saved_err & UIC_ERROR) &&
- (hba->saved_uic_err & (UFSHCD_UIC_DL_PA_INIT_ERROR |
- UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
- UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
+ (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
+ UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
needs_reset = true;
+ if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
+ (hba->saved_uic_err &&
+ (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
+ bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
+
+ spin_unlock_irqrestore(hba->host->host_lock, flags);
+ ufshcd_print_host_state(hba);
+ ufshcd_print_pwr_info(hba);
+ ufshcd_print_host_regs(hba);
+ ufshcd_print_tmrs(hba, hba->outstanding_tasks);
+ ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt);
+ spin_lock_irqsave(hba->host->host_lock, flags);
+ }
+
/*
* if host reset is required then skip clearing the pending
* transfers forcefully because they will get cleared during
* host reset and restore
*/
if (needs_reset)
- goto skip_pending_xfer_clear;
+ goto do_reset;
+ /*
+ * If LINERESET was caught, UFS might have been put to PWM mode,
+ * check if power mode restore is needed.
+ */
+ if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
+ hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
+ if (!hba->saved_uic_err)
+ hba->saved_err &= ~UIC_ERROR;
+ spin_unlock_irqrestore(hba->host->host_lock, flags);
+ if (ufshcd_is_pwr_mode_restore_needed(hba))
+ needs_restore = true;
+ spin_lock_irqsave(hba->host->host_lock, flags);
+ if (!hba->saved_err && !needs_restore)
+ goto skip_err_handling;
+ }
+
+ hba->silence_err_logs = true;
/* release lock as clear command might sleep */
spin_unlock_irqrestore(hba->host->host_lock, flags);
/* Clear pending transfer requests */
for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
- if (ufshcd_clear_cmd(hba, tag)) {
+ if (ufshcd_try_to_abort_task(hba, tag)) {
err_xfer = true;
goto lock_skip_pending_xfer_clear;
}
@@ -5632,11 +5810,38 @@ lock_skip_pending_xfer_clear:
/* Complete the requests that are cleared by s/w */
ufshcd_complete_requests(hba);
+ hba->silence_err_logs = false;
- if (err_xfer || err_tm)
+ if (err_xfer || err_tm) {
needs_reset = true;
+ goto do_reset;
+ }
-skip_pending_xfer_clear:
+ /*
+ * After all reqs and tasks are cleared from doorbell,
+ * now it is safe to retore power mode.
+ */
+ if (needs_restore) {
+ spin_unlock_irqrestore(hba->host->host_lock, flags);
+ /*
+ * Hold the scaling lock just in case dev cmds
+ * are sent via bsg and/or sysfs.
+ */
+ down_write(&hba->clk_scaling_lock);
+ hba->force_pmc = true;
+ pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
+ if (pmc_err) {
+ needs_reset = true;
+ dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
+ __func__, pmc_err);
+ }
+ hba->force_pmc = false;
+ ufshcd_print_pwr_info(hba);
+ up_write(&hba->clk_scaling_lock);
+ spin_lock_irqsave(hba->host->host_lock, flags);
+ }
+
+do_reset:
/* Fatal errors need reset */
if (needs_reset) {
unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
@@ -5652,38 +5857,31 @@ skip_pending_xfer_clear:
__ufshcd_transfer_req_compl(hba,
(1UL << (hba->nutrs - 1)));
+ hba->force_reset = false;
spin_unlock_irqrestore(hba->host->host_lock, flags);
err = ufshcd_reset_and_restore(hba);
+ if (err)
+ dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
+ __func__, err);
+ else
+ ufshcd_recover_pm_error(hba);
spin_lock_irqsave(hba->host->host_lock, flags);
- if (err) {
- dev_err(hba->dev, "%s: reset and restore failed\n",
- __func__);
- hba->ufshcd_state = UFSHCD_STATE_ERROR;
- }
- /*
- * Inform scsi mid-layer that we did reset and allow to handle
- * Unit Attention properly.
- */
- scsi_report_bus_reset(hba->host, 0);
- hba->saved_err = 0;
- hba->saved_uic_err = 0;
}
skip_err_handling:
if (!needs_reset) {
- hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
+ if (hba->ufshcd_state == UFSHCD_STATE_RESET)
+ hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
if (hba->saved_err || hba->saved_uic_err)
dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
__func__, hba->saved_err, hba->saved_uic_err);
}
- ufshcd_clear_eh_in_progress(hba);
-
out:
+ ufshcd_clear_eh_in_progress(hba);
spin_unlock_irqrestore(hba->host->host_lock, flags);
ufshcd_scsi_unblock_requests(hba);
- ufshcd_release(hba);
- pm_runtime_put_sync(hba->dev);
+ ufshcd_err_handling_unprepare(hba);
}
/**
@@ -5699,17 +5897,33 @@ static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
u32 reg;
irqreturn_t retval = IRQ_NONE;
- /* PHY layer lane error */
+ /* PHY layer error */
reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
- /* Ignore LINERESET indication, as this is not an error */
if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
- (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)) {
+ (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
+ ufshcd_update_reg_hist(&hba->ufs_stats.pa_err, reg);
/*
* To know whether this error is fatal or not, DB timeout
* must be checked but this error is handled separately.
*/
- dev_dbg(hba->dev, "%s: UIC Lane error reported\n", __func__);
- ufshcd_update_reg_hist(&hba->ufs_stats.pa_err, reg);
+ if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
+ dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
+ __func__);
+
+ /* Got a LINERESET indication. */
+ if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
+ struct uic_command *cmd = NULL;
+
+ hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
+ if (hba->uic_async_done && hba->active_uic_cmd)
+ cmd = hba->active_uic_cmd;
+ /*
+ * Ignore the LINERESET during power mode change
+ * operation via DME_SET command.
+ */
+ if (cmd && (cmd->command == UIC_CMD_DME_SET))
+ hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
+ }
retval |= IRQ_HANDLED;
}
@@ -5813,6 +6027,7 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba)
hba->errors, ufshcd_get_upmcrs(hba));
ufshcd_update_reg_hist(&hba->ufs_stats.auto_hibern8_err,
hba->errors);
+ ufshcd_set_link_broken(hba);
queue_eh_work = true;
}
@@ -5824,30 +6039,18 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba)
hba->saved_err |= hba->errors;
hba->saved_uic_err |= hba->uic_error;
- /* handle fatal errors only when link is functional */
- if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) {
- /* block commands from scsi mid-layer */
- ufshcd_scsi_block_requests(hba);
-
- hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED;
-
- /* dump controller state before resetting */
- if (hba->saved_err & (INT_FATAL_ERRORS | UIC_ERROR)) {
- bool pr_prdt = !!(hba->saved_err &
- SYSTEM_BUS_FATAL_ERROR);
-
- dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
+ /* dump controller state before resetting */
+ if ((hba->saved_err & (INT_FATAL_ERRORS)) ||
+ (hba->saved_uic_err &&
+ (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
+ dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
__func__, hba->saved_err,
hba->saved_uic_err);
-
- ufshcd_print_host_regs(hba);
- ufshcd_print_pwr_info(hba);
- ufshcd_print_tmrs(hba, hba->outstanding_tasks);
- ufshcd_print_trs(hba, hba->outstanding_reqs,
- pr_prdt);
- }
- schedule_work(&hba->eh_work);
+ ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
+ "host_regs: ");
+ ufshcd_print_pwr_info(hba);
}
+ ufshcd_schedule_eh_work(hba);
retval |= IRQ_HANDLED;
}
/*
@@ -5951,6 +6154,8 @@ static irqreturn_t ufshcd_intr(int irq, void *__hba)
spin_lock(hba->host->host_lock);
intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
+ hba->ufs_stats.last_intr_status = intr_status;
+ hba->ufs_stats.last_intr_ts = ktime_get();
/*
* There could be max of hba->nutrs reqs in flight and in worst case
@@ -6383,7 +6588,7 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
}
/**
- * ufshcd_abort - abort a specific command
+ * ufshcd_try_to_abort_task - abort a specific task
* @cmd: SCSI command pointer
*
* Abort the pending command in device by sending UFS_ABORT_TASK task management
@@ -6392,6 +6597,80 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
* issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
* really issued and then try to abort it.
*
+ * Returns zero on success, non-zero on failure
+ */
+static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
+{
+ struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ int err = 0;
+ int poll_cnt;
+ u8 resp = 0xF;
+ u32 reg;
+
+ for (poll_cnt = 100; poll_cnt; poll_cnt--) {
+ err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
+ UFS_QUERY_TASK, &resp);
+ if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
+ /* cmd pending in the device */
+ dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
+ __func__, tag);
+ break;
+ } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
+ /*
+ * cmd not pending in the device, check if it is
+ * in transition.
+ */
+ dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
+ __func__, tag);
+ reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
+ if (reg & (1 << tag)) {
+ /* sleep for max. 200us to stabilize */
+ usleep_range(100, 200);
+ continue;
+ }
+ /* command completed already */
+ dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
+ __func__, tag);
+ goto out;
+ } else {
+ dev_err(hba->dev,
+ "%s: no response from device. tag = %d, err %d\n",
+ __func__, tag, err);
+ if (!err)
+ err = resp; /* service response error */
+ goto out;
+ }
+ }
+
+ if (!poll_cnt) {
+ err = -EBUSY;
+ goto out;
+ }
+
+ err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
+ UFS_ABORT_TASK, &resp);
+ if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
+ if (!err) {
+ err = resp; /* service response error */
+ dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
+ __func__, tag, err);
+ }
+ goto out;
+ }
+
+ err = ufshcd_clear_cmd(hba, tag);
+ if (err)
+ dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
+ __func__, tag, err);
+
+out:
+ return err;
+}
+
+/**
+ * ufshcd_abort - scsi host template eh_abort_handler callback
+ * @cmd: SCSI command pointer
+ *
* Returns SUCCESS/FAILED
*/
static int ufshcd_abort(struct scsi_cmnd *cmd)
@@ -6401,8 +6680,6 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
unsigned long flags;
unsigned int tag;
int err = 0;
- int poll_cnt;
- u8 resp = 0xF;
struct ufshcd_lrb *lrbp;
u32 reg;
@@ -6467,79 +6744,17 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
}
/* Skip task abort in case previous aborts failed and report failure */
- if (lrbp->req_abort_skip) {
+ if (lrbp->req_abort_skip)
err = -EIO;
- goto out;
- }
-
- for (poll_cnt = 100; poll_cnt; poll_cnt--) {
- err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
- UFS_QUERY_TASK, &resp);
- if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
- /* cmd pending in the device */
- dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
- __func__, tag);
- break;
- } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
- /*
- * cmd not pending in the device, check if it is
- * in transition.
- */
- dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
- __func__, tag);
- reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
- if (reg & (1 << tag)) {
- /* sleep for max. 200us to stabilize */
- usleep_range(100, 200);
- continue;
- }
- /* command completed already */
- dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
- __func__, tag);
- goto cleanup;
- } else {
- dev_err(hba->dev,
- "%s: no response from device. tag = %d, err %d\n",
- __func__, tag, err);
- if (!err)
- err = resp; /* service response error */
- goto out;
- }
- }
-
- if (!poll_cnt) {
- err = -EBUSY;
- goto out;
- }
-
- err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
- UFS_ABORT_TASK, &resp);
- if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
- if (!err) {
- err = resp; /* service response error */
- dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
- __func__, tag, err);
- }
- goto out;
- }
-
- err = ufshcd_clear_cmd(hba, tag);
- if (err) {
- dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
- __func__, tag, err);
- goto out;
- }
+ else
+ err = ufshcd_try_to_abort_task(hba, tag);
+ if (!err) {
cleanup:
- scsi_dma_unmap(cmd);
-
- spin_lock_irqsave(host->host_lock, flags);
- ufshcd_outstanding_req_clear(hba, tag);
- hba->lrb[tag].cmd = NULL;
- spin_unlock_irqrestore(host->host_lock, flags);
-
+ spin_lock_irqsave(host->host_lock, flags);
+ __ufshcd_transfer_req_compl(hba, (1UL << tag));
+ spin_unlock_irqrestore(host->host_lock, flags);
out:
- if (!err) {
err = SUCCESS;
} else {
dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
@@ -6592,8 +6807,6 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
/* Establish the link again and restore the device */
err = ufshcd_probe_hba(hba, false);
- if (!err && (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL))
- err = -EIO;
out:
if (err)
dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
@@ -6612,9 +6825,23 @@ out:
*/
static int ufshcd_reset_and_restore(struct ufs_hba *hba)
{
+ u32 saved_err;
+ u32 saved_uic_err;
int err = 0;
+ unsigned long flags;
int retries = MAX_HOST_RESET_RETRIES;
+ /*
+ * This is a fresh start, cache and clear saved error first,
+ * in case new error generated during reset and restore.
+ */
+ spin_lock_irqsave(hba->host->host_lock, flags);
+ saved_err = hba->saved_err;
+ saved_uic_err = hba->saved_uic_err;
+ hba->saved_err = 0;
+ hba->saved_uic_err = 0;
+ spin_unlock_irqrestore(hba->host->host_lock, flags);
+
do {
/* Reset the attached device */
ufshcd_vops_device_reset(hba);
@@ -6622,6 +6849,18 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba)
err = ufshcd_host_reset_and_restore(hba);
} while (err && --retries);
+ spin_lock_irqsave(hba->host->host_lock, flags);
+ /*
+ * Inform scsi mid-layer that we did reset and allow to handle
+ * Unit Attention properly.
+ */
+ scsi_report_bus_reset(hba->host, 0);
+ if (err) {
+ hba->saved_err |= saved_err;
+ hba->saved_uic_err |= saved_uic_err;
+ }
+ spin_unlock_irqrestore(hba->host->host_lock, flags);
+
return err;
}
@@ -6633,48 +6872,25 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba)
*/
static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
{
- int err;
+ int err = SUCCESS;
unsigned long flags;
struct ufs_hba *hba;
hba = shost_priv(cmd->device->host);
- ufshcd_hold(hba, false);
- /*
- * Check if there is any race with fatal error handling.
- * If so, wait for it to complete. Even though fatal error
- * handling does reset and restore in some cases, don't assume
- * anything out of it. We are just avoiding race here.
- */
- do {
- spin_lock_irqsave(hba->host->host_lock, flags);
- if (!(work_pending(&hba->eh_work) ||
- hba->ufshcd_state == UFSHCD_STATE_RESET ||
- hba->ufshcd_state == UFSHCD_STATE_EH_SCHEDULED))
- break;
- spin_unlock_irqrestore(hba->host->host_lock, flags);
- dev_dbg(hba->dev, "%s: reset in progress\n", __func__);
- flush_work(&hba->eh_work);
- } while (1);
-
- hba->ufshcd_state = UFSHCD_STATE_RESET;
- ufshcd_set_eh_in_progress(hba);
+ spin_lock_irqsave(hba->host->host_lock, flags);
+ hba->force_reset = true;
+ ufshcd_schedule_eh_work(hba);
+ dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
spin_unlock_irqrestore(hba->host->host_lock, flags);
- err = ufshcd_reset_and_restore(hba);
+ flush_work(&hba->eh_work);
spin_lock_irqsave(hba->host->host_lock, flags);
- if (!err) {
- err = SUCCESS;
- hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
- } else {
+ if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
err = FAILED;
- hba->ufshcd_state = UFSHCD_STATE_ERROR;
- }
- ufshcd_clear_eh_in_progress(hba);
spin_unlock_irqrestore(hba->host->host_lock, flags);
- ufshcd_release(hba);
return err;
}
@@ -7395,6 +7611,7 @@ out:
static int ufshcd_probe_hba(struct ufs_hba *hba, bool async)
{
int ret;
+ unsigned long flags;
ktime_t start = ktime_get();
ret = ufshcd_link_startup(hba);
@@ -7459,14 +7676,17 @@ static int ufshcd_probe_hba(struct ufs_hba *hba, bool async)
*/
ufshcd_set_active_icc_lvl(hba);
- /* set the state as operational after switching to desired gear */
- hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
-
ufshcd_wb_config(hba);
/* Enable Auto-Hibernate if configured */
ufshcd_auto_hibern8_enable(hba);
out:
+ spin_lock_irqsave(hba->host->host_lock, flags);
+ if (ret)
+ hba->ufshcd_state = UFSHCD_STATE_ERROR;
+ else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
+ hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
+ spin_unlock_irqrestore(hba->host->host_lock, flags);
trace_ufshcd_init(dev_name(hba->dev), ret,
ktime_to_us(ktime_sub(ktime_get(), start)),
@@ -7606,12 +7826,10 @@ static int ufshcd_config_vreg(struct device *dev,
if (vreg->min_uV && vreg->max_uV) {
min_uV = on ? vreg->min_uV : 0;
ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
- if (ret) {
+ if (ret)
dev_err(dev,
"%s: %s set voltage failed, err=%d\n",
__func__, name, ret);
- goto out;
- }
}
}
out:
@@ -7674,8 +7892,6 @@ static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
goto out;
ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
- if (ret)
- goto out;
out:
if (ret) {
@@ -7721,10 +7937,8 @@ static int ufshcd_init_vreg(struct ufs_hba *hba)
goto out;
ret = ufshcd_get_vreg(dev, info->vccq);
- if (ret)
- goto out;
-
- ret = ufshcd_get_vreg(dev, info->vccq2);
+ if (!ret)
+ ret = ufshcd_get_vreg(dev, info->vccq2);
out:
return ret;
}
@@ -7868,12 +8082,7 @@ static int ufshcd_variant_hba_init(struct ufs_hba *hba)
err = ufshcd_vops_setup_regulators(hba, true);
if (err)
- goto out_exit;
-
- goto out;
-
-out_exit:
- ufshcd_vops_exit(hba);
+ ufshcd_vops_exit(hba);
out:
if (err)
dev_err(hba->dev, "%s: variant %s init failed err %d\n",
@@ -8073,10 +8282,13 @@ static int ufshcd_link_state_transition(struct ufs_hba *hba,
if (req_link_state == UIC_LINK_HIBERN8_STATE) {
ret = ufshcd_uic_hibern8_enter(hba);
- if (!ret)
+ if (!ret) {
ufshcd_set_link_hibern8(hba);
- else
+ } else {
+ dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
+ __func__, ret);
goto out;
+ }
}
/*
* If autobkops is enabled, link can't be turned off because
@@ -8092,8 +8304,11 @@ static int ufshcd_link_state_transition(struct ufs_hba *hba,
* unipro. But putting the link in hibern8 is much faster.
*/
ret = ufshcd_uic_hibern8_enter(hba);
- if (ret)
+ if (ret) {
+ dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
+ __func__, ret);
goto out;
+ }
/*
* Change controller state to "reset state" which
* should also put the link in off/reset state
@@ -8331,8 +8546,11 @@ disable_clks:
/* If link is active, device ref_clk can't be switched off */
__ufshcd_setup_clocks(hba, false, true);
- hba->clk_gating.state = CLKS_OFF;
- trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
+ if (ufshcd_is_clkgating_allowed(hba)) {
+ hba->clk_gating.state = CLKS_OFF;
+ trace_ufshcd_clk_gating(dev_name(hba->dev),
+ hba->clk_gating.state);
+ }
/* Put the host controller in low power mode if possible */
ufshcd_hba_vreg_set_lpm(hba);
@@ -8410,10 +8628,13 @@ static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
if (ufshcd_is_link_hibern8(hba)) {
ret = ufshcd_uic_hibern8_exit(hba);
- if (!ret)
+ if (!ret) {
ufshcd_set_link_active(hba);
- else
+ } else {
+ dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
+ __func__, ret);
goto vendor_suspend;
+ }
} else if (ufshcd_is_link_off(hba)) {
/*
* A full initialization of the host and the device is
@@ -8472,6 +8693,11 @@ disable_irq_and_vops_clks:
if (hba->clk_scaling.is_allowed)
ufshcd_suspend_clkscaling(hba);
ufshcd_setup_clocks(hba, false);
+ if (ufshcd_is_clkgating_allowed(hba)) {
+ hba->clk_gating.state = CLKS_OFF;
+ trace_ufshcd_clk_gating(dev_name(hba->dev),
+ hba->clk_gating.state);
+ }
out:
hba->pm_op_in_progress = 0;
if (ret)
@@ -8782,6 +9008,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
int err;
struct Scsi_Host *host = hba->host;
struct device *dev = hba->dev;
+ char eh_wq_name[sizeof("ufs_eh_wq_00")];
if (!mmio_base) {
dev_err(hba->dev,
@@ -8843,6 +9070,15 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
hba->max_pwr_info.is_valid = false;
/* Initialize work queues */
+ snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d",
+ hba->host->host_no);
+ hba->eh_wq = create_singlethread_workqueue(eh_wq_name);
+ if (!hba->eh_wq) {
+ dev_err(hba->dev, "%s: failed to create eh workqueue\n",
+ __func__);
+ err = -ENOMEM;
+ goto out_disable;
+ }
INIT_WORK(&hba->eh_work, ufshcd_err_handler);
INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
diff --git a/drivers/scsi/ufs/ufshcd.h b/drivers/scsi/ufs/ufshcd.h
index 363589c0bd37..47eb1430274c 100644
--- a/drivers/scsi/ufs/ufshcd.h
+++ b/drivers/scsi/ufs/ufshcd.h
@@ -90,6 +90,7 @@ enum uic_link_state {
UIC_LINK_OFF_STATE = 0, /* Link powered down or disabled */
UIC_LINK_ACTIVE_STATE = 1, /* Link is in Fast/Slow/Sleep state */
UIC_LINK_HIBERN8_STATE = 2, /* Link is in Hibernate state */
+ UIC_LINK_BROKEN_STATE = 3, /* Link is in broken state */
};
#define ufshcd_is_link_off(hba) ((hba)->uic_link_state == UIC_LINK_OFF_STATE)
@@ -97,11 +98,15 @@ enum uic_link_state {
UIC_LINK_ACTIVE_STATE)
#define ufshcd_is_link_hibern8(hba) ((hba)->uic_link_state == \
UIC_LINK_HIBERN8_STATE)
+#define ufshcd_is_link_broken(hba) ((hba)->uic_link_state == \
+ UIC_LINK_BROKEN_STATE)
#define ufshcd_set_link_off(hba) ((hba)->uic_link_state = UIC_LINK_OFF_STATE)
#define ufshcd_set_link_active(hba) ((hba)->uic_link_state = \
UIC_LINK_ACTIVE_STATE)
#define ufshcd_set_link_hibern8(hba) ((hba)->uic_link_state = \
UIC_LINK_HIBERN8_STATE)
+#define ufshcd_set_link_broken(hba) ((hba)->uic_link_state = \
+ UIC_LINK_BROKEN_STATE)
#define ufshcd_set_ufs_dev_active(h) \
((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
@@ -409,6 +414,8 @@ struct ufs_err_reg_hist {
/**
* struct ufs_stats - keeps usage/err statistics
+ * @last_intr_status: record the last interrupt status.
+ * @last_intr_ts: record the last interrupt timestamp.
* @hibern8_exit_cnt: Counter to keep track of number of exits,
* reset this after link-startup.
* @last_hibern8_exit_tstamp: Set time after the hibern8 exit.
@@ -428,6 +435,9 @@ struct ufs_err_reg_hist {
* @tsk_abort: tracks task abort events
*/
struct ufs_stats {
+ u32 last_intr_status;
+ ktime_t last_intr_ts;
+
u32 hibern8_exit_cnt;
ktime_t last_hibern8_exit_tstamp;
@@ -526,6 +536,12 @@ enum ufshcd_quirks {
* auto-hibernate capability but it doesn't work.
*/
UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8 = 1 << 11,
+
+ /*
+ * This quirk needs to disable manual flush for write booster
+ */
+ UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL = 1 << 12,
+
};
enum ufshcd_caps {
@@ -617,12 +633,15 @@ struct ufs_hba_variant_params {
* @intr_mask: Interrupt Mask Bits
* @ee_ctrl_mask: Exception event control mask
* @is_powered: flag to check if HBA is powered
+ * @eh_wq: Workqueue that eh_work works on
* @eh_work: Worker to handle UFS errors that require s/w attention
* @eeh_work: Worker to handle exception events
* @errors: HBA errors
* @uic_error: UFS interconnect layer error status
* @saved_err: sticky error mask
* @saved_uic_err: sticky UIC error mask
+ * @force_reset: flag to force eh_work perform a full reset
+ * @force_pmc: flag to force a power mode change
* @silence_err_logs: flag to silence error logs
* @dev_cmd: ufs device management command information
* @last_dme_cmd_tstamp: time stamp of the last completed DME command
@@ -711,6 +730,7 @@ struct ufs_hba {
bool is_powered;
/* Work Queues */
+ struct workqueue_struct *eh_wq;
struct work_struct eh_work;
struct work_struct eeh_work;
@@ -720,6 +740,8 @@ struct ufs_hba {
u32 saved_err;
u32 saved_uic_err;
struct ufs_stats ufs_stats;
+ bool force_reset;
+ bool force_pmc;
bool silence_err_logs;
/* Device management request data */
diff --git a/drivers/scsi/ufs/ufshci.h b/drivers/scsi/ufs/ufshci.h
index ba31b090f784..6795e1f0e8f8 100644
--- a/drivers/scsi/ufs/ufshci.h
+++ b/drivers/scsi/ufs/ufshci.h
@@ -171,6 +171,7 @@ enum {
#define UIC_PHY_ADAPTER_LAYER_ERROR 0x80000000
#define UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK 0x1F
#define UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK 0xF
+#define UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR 0x10
/* UECDL - Host UIC Error Code Data Link Layer 3Ch */
#define UIC_DATA_LINK_LAYER_ERROR 0x80000000
diff --git a/drivers/scsi/ufs/unipro.h b/drivers/scsi/ufs/unipro.h
index 4ee64782fd48..f6b52ce36de6 100644
--- a/drivers/scsi/ufs/unipro.h
+++ b/drivers/scsi/ufs/unipro.h
@@ -205,6 +205,9 @@ enum {
UNCHANGED = 7,
};
+#define PWRMODE_MASK 0xF
+#define PWRMODE_RX_OFFSET 4
+
/* PA TX/RX Frequency Series */
enum {
PA_HS_MODE_A = 1,
generated by cgit v1.2.3 (git 2.39.0) at 2024-09-12 19:06:30 +0300