summaryrefslogtreecommitdiff
path: root/drivers/dma/idxd
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2022-05-29 21:38:27 +0300
committerLinus Torvalds <torvalds@linux-foundation.org>2022-05-29 21:38:27 +0300
commitb00ed48bb0a7c295facf9036135a573a5cdbe7de (patch)
treea14d3702971e30dfec8392af4f7b9e97d2246f95 /drivers/dma/idxd
parentc3a9a3c5f5590e85da15d6201e415ff636fe5670 (diff)
parentd1a28597808268b87f156138aad3104aa255e62b (diff)
downloadlinux-b00ed48bb0a7c295facf9036135a573a5cdbe7de.tar.xz
Merge tag 'dmaengine-5.19-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine
Pull dmaengine updates from Vinod Koul: "Nothing special, this includes a couple of new device support and new driver support and bunch of driver updates. New support: - Tegra gpcdma driver support - Qualcomm SM8350, Sm8450 and SC7280 device support - Renesas RZN1 dma and platform support Updates: - stm32 device pause/resume support and updates - DMA memset ops Documentation and usage clarification - deprecate '#dma-channels' & '#dma-requests' bindings - driver updates for stm32, ptdma idsx etc" * tag 'dmaengine-5.19-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine: (87 commits) dmaengine: idxd: make idxd_wq_enable() return 0 if wq is already enabled dmaengine: sun6i: Add support for the D1 variant dmaengine: sun6i: Add support for 34-bit physical addresses dmaengine: sun6i: Do not use virt_to_phys dt-bindings: dma: sun50i-a64: Add compatible for D1 dmaengine: tegra: Remove unused switch case dmaengine: tegra: Fix uninitialized variable usage dmaengine: stm32-dma: add device_pause/device_resume support dmaengine: stm32-dma: rename pm ops before dma pause/resume introduction dmaengine: stm32-dma: pass DMA_SxSCR value to stm32_dma_handle_chan_done() dmaengine: stm32-dma: introduce stm32_dma_sg_inc to manage chan->next_sg dmaengine: stm32-dmamux: avoid reset of dmamux if used by coprocessor dmaengine: qcom: gpi: Add support for sc7280 dt-bindings: dma: pl330: Add power-domains dmaengine: stm32-mdma: use dev_dbg on non-busy channel spurious it dmaengine: stm32-mdma: fix chan initialization in stm32_mdma_irq_handler() dmaengine: stm32-mdma: remove GISR1 register dmaengine: ti: deprecate '#dma-channels' dmaengine: mmp: deprecate '#dma-channels' dmaengine: pxa: deprecate '#dma-channels' and '#dma-requests' ...
Diffstat (limited to 'drivers/dma/idxd')
-rw-r--r--drivers/dma/idxd/cdev.c18
-rw-r--r--drivers/dma/idxd/device.c151
-rw-r--r--drivers/dma/idxd/dma.c65
-rw-r--r--drivers/dma/idxd/idxd.h20
-rw-r--r--drivers/dma/idxd/init.c30
-rw-r--r--drivers/dma/idxd/registers.h1
-rw-r--r--drivers/dma/idxd/sysfs.c12
7 files changed, 184 insertions, 113 deletions
diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c
index b9b2b4a4124e..c2808fd081d6 100644
--- a/drivers/dma/idxd/cdev.c
+++ b/drivers/dma/idxd/cdev.c
@@ -99,7 +99,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp)
ctx->wq = wq;
filp->private_data = ctx;
- if (device_pasid_enabled(idxd)) {
+ if (device_user_pasid_enabled(idxd)) {
sva = iommu_sva_bind_device(dev, current->mm, NULL);
if (IS_ERR(sva)) {
rc = PTR_ERR(sva);
@@ -152,7 +152,7 @@ static int idxd_cdev_release(struct inode *node, struct file *filep)
if (wq_shared(wq)) {
idxd_device_drain_pasid(idxd, ctx->pasid);
} else {
- if (device_pasid_enabled(idxd)) {
+ if (device_user_pasid_enabled(idxd)) {
/* The wq disable in the disable pasid function will drain the wq */
rc = idxd_wq_disable_pasid(wq);
if (rc < 0)
@@ -314,7 +314,7 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)
mutex_lock(&wq->wq_lock);
wq->type = IDXD_WQT_USER;
- rc = __drv_enable_wq(wq);
+ rc = drv_enable_wq(wq);
if (rc < 0)
goto err;
@@ -329,7 +329,7 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)
return 0;
err_cdev:
- __drv_disable_wq(wq);
+ drv_disable_wq(wq);
err:
wq->type = IDXD_WQT_NONE;
mutex_unlock(&wq->wq_lock);
@@ -342,7 +342,7 @@ static void idxd_user_drv_remove(struct idxd_dev *idxd_dev)
mutex_lock(&wq->wq_lock);
idxd_wq_del_cdev(wq);
- __drv_disable_wq(wq);
+ drv_disable_wq(wq);
wq->type = IDXD_WQT_NONE;
mutex_unlock(&wq->wq_lock);
}
@@ -369,10 +369,16 @@ int idxd_cdev_register(void)
rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
ictx[i].name);
if (rc)
- return rc;
+ goto err_free_chrdev_region;
}
return 0;
+
+err_free_chrdev_region:
+ for (i--; i >= 0; i--)
+ unregister_chrdev_region(ictx[i].devt, MINORMASK);
+
+ return rc;
}
void idxd_cdev_remove(void)
diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c
index f652da6ab47d..ff0ea60051f0 100644
--- a/drivers/dma/idxd/device.c
+++ b/drivers/dma/idxd/device.c
@@ -184,7 +184,7 @@ int idxd_wq_enable(struct idxd_wq *wq)
if (wq->state == IDXD_WQ_ENABLED) {
dev_dbg(dev, "WQ %d already enabled\n", wq->id);
- return -ENXIO;
+ return 0;
}
idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status);
@@ -299,24 +299,46 @@ void idxd_wqs_unmap_portal(struct idxd_device *idxd)
}
}
-int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)
+static void __idxd_wq_set_priv_locked(struct idxd_wq *wq, int priv)
{
struct idxd_device *idxd = wq->idxd;
- int rc;
union wqcfg wqcfg;
unsigned int offset;
- rc = idxd_wq_disable(wq, false);
- if (rc < 0)
- return rc;
+ offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PRIVL_IDX);
+ spin_lock(&idxd->dev_lock);
+ wqcfg.bits[WQCFG_PRIVL_IDX] = ioread32(idxd->reg_base + offset);
+ wqcfg.priv = priv;
+ wq->wqcfg->bits[WQCFG_PRIVL_IDX] = wqcfg.bits[WQCFG_PRIVL_IDX];
+ iowrite32(wqcfg.bits[WQCFG_PRIVL_IDX], idxd->reg_base + offset);
+ spin_unlock(&idxd->dev_lock);
+}
+
+static void __idxd_wq_set_pasid_locked(struct idxd_wq *wq, int pasid)
+{
+ struct idxd_device *idxd = wq->idxd;
+ union wqcfg wqcfg;
+ unsigned int offset;
offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);
spin_lock(&idxd->dev_lock);
wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset);
wqcfg.pasid_en = 1;
wqcfg.pasid = pasid;
+ wq->wqcfg->bits[WQCFG_PASID_IDX] = wqcfg.bits[WQCFG_PASID_IDX];
iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset);
spin_unlock(&idxd->dev_lock);
+}
+
+int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)
+{
+ int rc;
+
+ rc = idxd_wq_disable(wq, false);
+ if (rc < 0)
+ return rc;
+
+ __idxd_wq_set_pasid_locked(wq, pasid);
rc = idxd_wq_enable(wq);
if (rc < 0)
@@ -555,19 +577,15 @@ int idxd_device_disable(struct idxd_device *idxd)
return -ENXIO;
}
- spin_lock(&idxd->dev_lock);
idxd_device_clear_state(idxd);
- idxd->state = IDXD_DEV_DISABLED;
- spin_unlock(&idxd->dev_lock);
return 0;
}
void idxd_device_reset(struct idxd_device *idxd)
{
idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL);
- spin_lock(&idxd->dev_lock);
idxd_device_clear_state(idxd);
- idxd->state = IDXD_DEV_DISABLED;
+ spin_lock(&idxd->dev_lock);
idxd_unmask_error_interrupts(idxd);
spin_unlock(&idxd->dev_lock);
}
@@ -694,15 +712,16 @@ static void idxd_device_wqs_clear_state(struct idxd_device *idxd)
{
int i;
- lockdep_assert_held(&idxd->dev_lock);
for (i = 0; i < idxd->max_wqs; i++) {
struct idxd_wq *wq = idxd->wqs[i];
+ mutex_lock(&wq->wq_lock);
if (wq->state == IDXD_WQ_ENABLED) {
idxd_wq_disable_cleanup(wq);
wq->state = IDXD_WQ_DISABLED;
}
idxd_wq_device_reset_cleanup(wq);
+ mutex_unlock(&wq->wq_lock);
}
}
@@ -711,9 +730,12 @@ void idxd_device_clear_state(struct idxd_device *idxd)
if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
return;
+ idxd_device_wqs_clear_state(idxd);
+ spin_lock(&idxd->dev_lock);
idxd_groups_clear_state(idxd);
idxd_engines_clear_state(idxd);
- idxd_device_wqs_clear_state(idxd);
+ idxd->state = IDXD_DEV_DISABLED;
+ spin_unlock(&idxd->dev_lock);
}
static void idxd_group_config_write(struct idxd_group *group)
@@ -799,7 +821,7 @@ static int idxd_wq_config_write(struct idxd_wq *wq)
*/
for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
wq_offset = WQCFG_OFFSET(idxd, wq->id, i);
- wq->wqcfg->bits[i] = ioread32(idxd->reg_base + wq_offset);
+ wq->wqcfg->bits[i] |= ioread32(idxd->reg_base + wq_offset);
}
if (wq->size == 0 && wq->type != IDXD_WQT_NONE)
@@ -815,14 +837,8 @@ static int idxd_wq_config_write(struct idxd_wq *wq)
if (wq_dedicated(wq))
wq->wqcfg->mode = 1;
- if (device_pasid_enabled(idxd)) {
- wq->wqcfg->pasid_en = 1;
- if (wq->type == IDXD_WQT_KERNEL && wq_dedicated(wq))
- wq->wqcfg->pasid = idxd->pasid;
- }
-
/*
- * Here the priv bit is set depending on the WQ type. priv = 1 if the
+ * The WQ priv bit is set depending on the WQ type. priv = 1 if the
* WQ type is kernel to indicate privileged access. This setting only
* matters for dedicated WQ. According to the DSA spec:
* If the WQ is in dedicated mode, WQ PASID Enable is 1, and the
@@ -832,7 +848,6 @@ static int idxd_wq_config_write(struct idxd_wq *wq)
* In the case of a dedicated kernel WQ that is not able to support
* the PASID cap, then the configuration will be rejected.
*/
- wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL);
if (wq_dedicated(wq) && wq->wqcfg->pasid_en &&
!idxd_device_pasid_priv_enabled(idxd) &&
wq->type == IDXD_WQT_KERNEL) {
@@ -953,7 +968,7 @@ static int idxd_wqs_setup(struct idxd_device *idxd)
if (!wq->group)
continue;
- if (wq_shared(wq) && !device_swq_supported(idxd)) {
+ if (wq_shared(wq) && !wq_shared_supported(wq)) {
idxd->cmd_status = IDXD_SCMD_WQ_NO_SWQ_SUPPORT;
dev_warn(dev, "No shared wq support but configured.\n");
return -EINVAL;
@@ -1018,6 +1033,9 @@ static int idxd_wq_load_config(struct idxd_wq *wq)
wq->priority = wq->wqcfg->priority;
+ wq->max_xfer_bytes = 1ULL << wq->wqcfg->max_xfer_shift;
+ wq->max_batch_size = 1ULL << wq->wqcfg->max_batch_shift;
+
for (i = 0; i < WQCFG_STRIDES(idxd); i++) {
wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i);
dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wqcfg_offset, wq->wqcfg->bits[i]);
@@ -1161,7 +1179,9 @@ void idxd_wq_free_irq(struct idxd_wq *wq)
struct idxd_device *idxd = wq->idxd;
struct idxd_irq_entry *ie = &wq->ie;
- synchronize_irq(ie->vector);
+ if (wq->type != IDXD_WQT_KERNEL)
+ return;
+
free_irq(ie->vector, ie);
idxd_flush_pending_descs(ie);
if (idxd->request_int_handles)
@@ -1180,6 +1200,9 @@ int idxd_wq_request_irq(struct idxd_wq *wq)
struct idxd_irq_entry *ie;
int rc;
+ if (wq->type != IDXD_WQT_KERNEL)
+ return 0;
+
ie = &wq->ie;
ie->vector = pci_irq_vector(pdev, ie->id);
ie->pasid = device_pasid_enabled(idxd) ? idxd->pasid : INVALID_IOASID;
@@ -1211,7 +1234,7 @@ err_irq:
return rc;
}
-int __drv_enable_wq(struct idxd_wq *wq)
+int drv_enable_wq(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
@@ -1245,7 +1268,7 @@ int __drv_enable_wq(struct idxd_wq *wq)
/* Shared WQ checks */
if (wq_shared(wq)) {
- if (!device_swq_supported(idxd)) {
+ if (!wq_shared_supported(wq)) {
idxd->cmd_status = IDXD_SCMD_WQ_NO_SVM;
dev_dbg(dev, "PASID not enabled and shared wq.\n");
goto err;
@@ -1265,6 +1288,29 @@ int __drv_enable_wq(struct idxd_wq *wq)
}
}
+ /*
+ * In the event that the WQ is configurable for pasid and priv bits.
+ * For kernel wq, the driver should setup the pasid, pasid_en, and priv bit.
+ * However, for non-kernel wq, the driver should only set the pasid_en bit for
+ * shared wq. A dedicated wq that is not 'kernel' type will configure pasid and
+ * pasid_en later on so there is no need to setup.
+ */
+ if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) {
+ int priv = 0;
+
+ if (wq_pasid_enabled(wq)) {
+ if (is_idxd_wq_kernel(wq) || wq_shared(wq)) {
+ u32 pasid = wq_dedicated(wq) ? idxd->pasid : 0;
+
+ __idxd_wq_set_pasid_locked(wq, pasid);
+ }
+ }
+
+ if (is_idxd_wq_kernel(wq))
+ priv = 1;
+ __idxd_wq_set_priv_locked(wq, priv);
+ }
+
rc = 0;
spin_lock(&idxd->dev_lock);
if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
@@ -1289,8 +1335,36 @@ int __drv_enable_wq(struct idxd_wq *wq)
}
wq->client_count = 0;
+
+ rc = idxd_wq_request_irq(wq);
+ if (rc < 0) {
+ idxd->cmd_status = IDXD_SCMD_WQ_IRQ_ERR;
+ dev_dbg(dev, "WQ %d irq setup failed: %d\n", wq->id, rc);
+ goto err_irq;
+ }
+
+ rc = idxd_wq_alloc_resources(wq);
+ if (rc < 0) {
+ idxd->cmd_status = IDXD_SCMD_WQ_RES_ALLOC_ERR;
+ dev_dbg(dev, "WQ resource alloc failed\n");
+ goto err_res_alloc;
+ }
+
+ rc = idxd_wq_init_percpu_ref(wq);
+ if (rc < 0) {
+ idxd->cmd_status = IDXD_SCMD_PERCPU_ERR;
+ dev_dbg(dev, "percpu_ref setup failed\n");
+ goto err_ref;
+ }
+
return 0;
+err_ref:
+ idxd_wq_free_resources(wq);
+err_res_alloc:
+ idxd_wq_free_irq(wq);
+err_irq:
+ idxd_wq_unmap_portal(wq);
err_map_portal:
rc = idxd_wq_disable(wq, false);
if (rc < 0)
@@ -1299,17 +1373,7 @@ err:
return rc;
}
-int drv_enable_wq(struct idxd_wq *wq)
-{
- int rc;
-
- mutex_lock(&wq->wq_lock);
- rc = __drv_enable_wq(wq);
- mutex_unlock(&wq->wq_lock);
- return rc;
-}
-
-void __drv_disable_wq(struct idxd_wq *wq)
+void drv_disable_wq(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct device *dev = &idxd->pdev->dev;
@@ -1320,21 +1384,16 @@ void __drv_disable_wq(struct idxd_wq *wq)
dev_warn(dev, "Clients has claim on wq %d: %d\n",
wq->id, idxd_wq_refcount(wq));
+ idxd_wq_free_resources(wq);
idxd_wq_unmap_portal(wq);
-
idxd_wq_drain(wq);
+ idxd_wq_free_irq(wq);
idxd_wq_reset(wq);
-
+ percpu_ref_exit(&wq->wq_active);
+ wq->type = IDXD_WQT_NONE;
wq->client_count = 0;
}
-void drv_disable_wq(struct idxd_wq *wq)
-{
- mutex_lock(&wq->wq_lock);
- __drv_disable_wq(wq);
- mutex_unlock(&wq->wq_lock);
-}
-
int idxd_device_drv_probe(struct idxd_dev *idxd_dev)
{
struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev);
diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c
index bfff59617d04..e0874cb4721c 100644
--- a/drivers/dma/idxd/dma.c
+++ b/drivers/dma/idxd/dma.c
@@ -88,6 +88,27 @@ static inline void idxd_prep_desc_common(struct idxd_wq *wq,
}
static struct dma_async_tx_descriptor *
+idxd_dma_prep_interrupt(struct dma_chan *c, unsigned long flags)
+{
+ struct idxd_wq *wq = to_idxd_wq(c);
+ u32 desc_flags;
+ struct idxd_desc *desc;
+
+ if (wq->state != IDXD_WQ_ENABLED)
+ return NULL;
+
+ op_flag_setup(flags, &desc_flags);
+ desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
+ if (IS_ERR(desc))
+ return NULL;
+
+ idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_NOOP,
+ 0, 0, 0, desc->compl_dma, desc_flags);
+ desc->txd.flags = flags;
+ return &desc->txd;
+}
+
+static struct dma_async_tx_descriptor *
idxd_dma_submit_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
dma_addr_t dma_src, size_t len, unsigned long flags)
{
@@ -193,10 +214,12 @@ int idxd_register_dma_device(struct idxd_device *idxd)
INIT_LIST_HEAD(&dma->channels);
dma->dev = dev;
+ dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
dma_cap_set(DMA_PRIVATE, dma->cap_mask);
dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask);
dma->device_release = idxd_dma_release;
+ dma->device_prep_dma_interrupt = idxd_dma_prep_interrupt;
if (idxd->hw.opcap.bits[0] & IDXD_OPCAP_MEMMOVE) {
dma_cap_set(DMA_MEMCPY, dma->cap_mask);
dma->device_prep_dma_memcpy = idxd_dma_submit_memcpy;
@@ -227,7 +250,7 @@ void idxd_unregister_dma_device(struct idxd_device *idxd)
dma_async_device_unregister(&idxd->idxd_dma->dma);
}
-int idxd_register_dma_channel(struct idxd_wq *wq)
+static int idxd_register_dma_channel(struct idxd_wq *wq)
{
struct idxd_device *idxd = wq->idxd;
struct dma_device *dma = &idxd->idxd_dma->dma;
@@ -264,7 +287,7 @@ int idxd_register_dma_channel(struct idxd_wq *wq)
return 0;
}
-void idxd_unregister_dma_channel(struct idxd_wq *wq)
+static void idxd_unregister_dma_channel(struct idxd_wq *wq)
{
struct idxd_dma_chan *idxd_chan = wq->idxd_chan;
struct dma_chan *chan = &idxd_chan->chan;
@@ -290,34 +313,13 @@ static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev)
mutex_lock(&wq->wq_lock);
wq->type = IDXD_WQT_KERNEL;
- rc = idxd_wq_request_irq(wq);
- if (rc < 0) {
- idxd->cmd_status = IDXD_SCMD_WQ_IRQ_ERR;
- dev_dbg(dev, "WQ %d irq setup failed: %d\n", wq->id, rc);
- goto err_irq;
- }
-
- rc = __drv_enable_wq(wq);
+ rc = drv_enable_wq(wq);
if (rc < 0) {
dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc);
rc = -ENXIO;
goto err;
}
- rc = idxd_wq_alloc_resources(wq);
- if (rc < 0) {
- idxd->cmd_status = IDXD_SCMD_WQ_RES_ALLOC_ERR;
- dev_dbg(dev, "WQ resource alloc failed\n");
- goto err_res_alloc;
- }
-
- rc = idxd_wq_init_percpu_ref(wq);
- if (rc < 0) {
- idxd->cmd_status = IDXD_SCMD_PERCPU_ERR;
- dev_dbg(dev, "percpu_ref setup failed\n");
- goto err_ref;
- }
-
rc = idxd_register_dma_channel(wq);
if (rc < 0) {
idxd->cmd_status = IDXD_SCMD_DMA_CHAN_ERR;
@@ -330,15 +332,8 @@ static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev)
return 0;
err_dma:
- __idxd_wq_quiesce(wq);
- percpu_ref_exit(&wq->wq_active);
-err_ref:
- idxd_wq_free_resources(wq);
-err_res_alloc:
- __drv_disable_wq(wq);
+ drv_disable_wq(wq);
err:
- idxd_wq_free_irq(wq);
-err_irq:
wq->type = IDXD_WQT_NONE;
mutex_unlock(&wq->wq_lock);
return rc;
@@ -351,11 +346,7 @@ static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev)
mutex_lock(&wq->wq_lock);
__idxd_wq_quiesce(wq);
idxd_unregister_dma_channel(wq);
- idxd_wq_free_resources(wq);
- __drv_disable_wq(wq);
- percpu_ref_exit(&wq->wq_active);
- idxd_wq_free_irq(wq);
- wq->type = IDXD_WQT_NONE;
+ drv_disable_wq(wq);
mutex_unlock(&wq->wq_lock);
}
diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h
index da72eb15f610..fed0dfc1eaa8 100644
--- a/drivers/dma/idxd/idxd.h
+++ b/drivers/dma/idxd/idxd.h
@@ -239,6 +239,7 @@ enum idxd_device_flag {
IDXD_FLAG_CONFIGURABLE = 0,
IDXD_FLAG_CMD_RUNNING,
IDXD_FLAG_PASID_ENABLED,
+ IDXD_FLAG_USER_PASID_ENABLED,
};
struct idxd_dma_dev {
@@ -469,9 +470,20 @@ static inline bool device_pasid_enabled(struct idxd_device *idxd)
return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
}
-static inline bool device_swq_supported(struct idxd_device *idxd)
+static inline bool device_user_pasid_enabled(struct idxd_device *idxd)
{
- return (support_enqcmd && device_pasid_enabled(idxd));
+ return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
+}
+
+static inline bool wq_pasid_enabled(struct idxd_wq *wq)
+{
+ return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) ||
+ (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd));
+}
+
+static inline bool wq_shared_supported(struct idxd_wq *wq)
+{
+ return (support_enqcmd && wq_pasid_enabled(wq));
}
enum idxd_portal_prot {
@@ -559,9 +571,7 @@ void idxd_unregister_idxd_drv(void);
int idxd_device_drv_probe(struct idxd_dev *idxd_dev);
void idxd_device_drv_remove(struct idxd_dev *idxd_dev);
int drv_enable_wq(struct idxd_wq *wq);
-int __drv_enable_wq(struct idxd_wq *wq);
void drv_disable_wq(struct idxd_wq *wq);
-void __drv_disable_wq(struct idxd_wq *wq);
int idxd_device_init_reset(struct idxd_device *idxd);
int idxd_device_enable(struct idxd_device *idxd);
int idxd_device_disable(struct idxd_device *idxd);
@@ -602,8 +612,6 @@ int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc);
/* dmaengine */
int idxd_register_dma_device(struct idxd_device *idxd);
void idxd_unregister_dma_device(struct idxd_device *idxd);
-int idxd_register_dma_channel(struct idxd_wq *wq);
-void idxd_unregister_dma_channel(struct idxd_wq *wq);
void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
void idxd_dma_complete_txd(struct idxd_desc *desc,
enum idxd_complete_type comp_type, bool free_desc);
diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c
index 993a5dcca24f..355fb3ef4cbf 100644
--- a/drivers/dma/idxd/init.c
+++ b/drivers/dma/idxd/init.c
@@ -512,18 +512,15 @@ static int idxd_probe(struct idxd_device *idxd)
dev_dbg(dev, "IDXD reset complete\n");
if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) {
- rc = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA);
- if (rc == 0) {
- rc = idxd_enable_system_pasid(idxd);
- if (rc < 0) {
- iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
- dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc);
- } else {
- set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
- }
- } else {
- dev_warn(dev, "Unable to turn on SVA feature.\n");
- }
+ if (iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA))
+ dev_warn(dev, "Unable to turn on user SVA feature.\n");
+ else
+ set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
+
+ if (idxd_enable_system_pasid(idxd))
+ dev_warn(dev, "No in-kernel DMA with PASID.\n");
+ else
+ set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
} else if (!sva) {
dev_warn(dev, "User forced SVA off via module param.\n");
}
@@ -561,7 +558,8 @@ static int idxd_probe(struct idxd_device *idxd)
err:
if (device_pasid_enabled(idxd))
idxd_disable_system_pasid(idxd);
- iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
+ if (device_user_pasid_enabled(idxd))
+ iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
return rc;
}
@@ -574,7 +572,8 @@ static void idxd_cleanup(struct idxd_device *idxd)
idxd_cleanup_internals(idxd);
if (device_pasid_enabled(idxd))
idxd_disable_system_pasid(idxd);
- iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
+ if (device_user_pasid_enabled(idxd))
+ iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA);
}
static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
@@ -691,7 +690,8 @@ static void idxd_remove(struct pci_dev *pdev)
free_irq(irq_entry->vector, irq_entry);
pci_free_irq_vectors(pdev);
pci_iounmap(pdev, idxd->reg_base);
- iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
+ if (device_user_pasid_enabled(idxd))
+ iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
pci_disable_device(pdev);
destroy_workqueue(idxd->wq);
perfmon_pmu_remove(idxd);
diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h
index aa642aecdc0b..02449aa9c454 100644
--- a/drivers/dma/idxd/registers.h
+++ b/drivers/dma/idxd/registers.h
@@ -353,6 +353,7 @@ union wqcfg {
} __packed;
#define WQCFG_PASID_IDX 2
+#define WQCFG_PRIVL_IDX 2
#define WQCFG_OCCUP_IDX 6
#define WQCFG_OCCUP_MASK 0xffff
diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c
index dfd549685c46..3f262a57441b 100644
--- a/drivers/dma/idxd/sysfs.c
+++ b/drivers/dma/idxd/sysfs.c
@@ -588,7 +588,7 @@ static ssize_t wq_mode_store(struct device *dev,
if (sysfs_streq(buf, "dedicated")) {
set_bit(WQ_FLAG_DEDICATED, &wq->flags);
wq->threshold = 0;
- } else if (sysfs_streq(buf, "shared") && device_swq_supported(idxd)) {
+ } else if (sysfs_streq(buf, "shared")) {
clear_bit(WQ_FLAG_DEDICATED, &wq->flags);
} else {
return -EINVAL;
@@ -832,6 +832,7 @@ static ssize_t wq_name_store(struct device *dev,
size_t count)
{
struct idxd_wq *wq = confdev_to_wq(dev);
+ char *input, *pos;
if (wq->state != IDXD_WQ_DISABLED)
return -EPERM;
@@ -846,9 +847,14 @@ static ssize_t wq_name_store(struct device *dev,
if (wq->type == IDXD_WQT_KERNEL && device_pasid_enabled(wq->idxd))
return -EOPNOTSUPP;
+ input = kstrndup(buf, count, GFP_KERNEL);
+ if (!input)
+ return -ENOMEM;
+
+ pos = strim(input);
memset(wq->name, 0, WQ_NAME_SIZE + 1);
- strncpy(wq->name, buf, WQ_NAME_SIZE);
- strreplace(wq->name, '\n', '\0');
+ sprintf(wq->name, "%s", pos);
+ kfree(input);
return count;
}