diff options
Diffstat (limited to 'drivers/dma/idxd')
-rw-r--r-- | drivers/dma/idxd/device.c | 161 | ||||
-rw-r--r-- | drivers/dma/idxd/dma.c | 12 | ||||
-rw-r--r-- | drivers/dma/idxd/idxd.h | 7 | ||||
-rw-r--r-- | drivers/dma/idxd/init.c | 133 | ||||
-rw-r--r-- | drivers/dma/idxd/irq.c | 3 |
5 files changed, 131 insertions, 185 deletions
diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 8233a29f859d..280b41417f41 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -19,36 +19,6 @@ static void idxd_device_wqs_clear_state(struct idxd_device *idxd); static void idxd_wq_disable_cleanup(struct idxd_wq *wq); /* Interrupt control bits */ -void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id) -{ - struct idxd_irq_entry *ie; - struct irq_data *data; - - ie = idxd_get_ie(idxd, vec_id); - data = irq_get_irq_data(ie->vector); - pci_msi_mask_irq(data); -} - -void idxd_mask_msix_vectors(struct idxd_device *idxd) -{ - struct pci_dev *pdev = idxd->pdev; - int msixcnt = pci_msix_vec_count(pdev); - int i; - - for (i = 0; i < msixcnt; i++) - idxd_mask_msix_vector(idxd, i); -} - -void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id) -{ - struct idxd_irq_entry *ie; - struct irq_data *data; - - ie = idxd_get_ie(idxd, vec_id); - data = irq_get_irq_data(ie->vector); - pci_msi_unmask_irq(data); -} - void idxd_unmask_error_interrupts(struct idxd_device *idxd) { union genctrl_reg genctrl; @@ -593,7 +563,6 @@ void idxd_device_reset(struct idxd_device *idxd) idxd_device_clear_state(idxd); idxd->state = IDXD_DEV_DISABLED; idxd_unmask_error_interrupts(idxd); - idxd_msix_perm_setup(idxd); spin_unlock(&idxd->dev_lock); } @@ -732,36 +701,6 @@ void idxd_device_clear_state(struct idxd_device *idxd) idxd_device_wqs_clear_state(idxd); } -void idxd_msix_perm_setup(struct idxd_device *idxd) -{ - union msix_perm mperm; - int i, msixcnt; - - msixcnt = pci_msix_vec_count(idxd->pdev); - if (msixcnt < 0) - return; - - mperm.bits = 0; - mperm.pasid = idxd->pasid; - mperm.pasid_en = device_pasid_enabled(idxd); - for (i = 1; i < msixcnt; i++) - iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8); -} - -void idxd_msix_perm_clear(struct idxd_device *idxd) -{ - union msix_perm mperm; - int i, msixcnt; - - msixcnt = pci_msix_vec_count(idxd->pdev); - if (msixcnt < 0) - return; - - mperm.bits = 0; - for (i = 1; i < msixcnt; i++) - iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8); -} - static void idxd_group_config_write(struct idxd_group *group) { struct idxd_device *idxd = group->idxd; @@ -1158,6 +1097,106 @@ int idxd_device_load_config(struct idxd_device *idxd) return 0; } +static void idxd_flush_pending_descs(struct idxd_irq_entry *ie) +{ + struct idxd_desc *desc, *itr; + struct llist_node *head; + LIST_HEAD(flist); + enum idxd_complete_type ctype; + + spin_lock(&ie->list_lock); + head = llist_del_all(&ie->pending_llist); + if (head) { + llist_for_each_entry_safe(desc, itr, head, llnode) + list_add_tail(&desc->list, &ie->work_list); + } + + list_for_each_entry_safe(desc, itr, &ie->work_list, list) + list_move_tail(&desc->list, &flist); + spin_unlock(&ie->list_lock); + + list_for_each_entry_safe(desc, itr, &flist, list) { + list_del(&desc->list); + ctype = desc->completion->status ? IDXD_COMPLETE_NORMAL : IDXD_COMPLETE_ABORT; + idxd_dma_complete_txd(desc, ctype, true); + } +} + +static void idxd_device_set_perm_entry(struct idxd_device *idxd, + struct idxd_irq_entry *ie) +{ + union msix_perm mperm; + + if (ie->pasid == INVALID_IOASID) + return; + + mperm.bits = 0; + mperm.pasid = ie->pasid; + mperm.pasid_en = 1; + iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + ie->id * 8); +} + +static void idxd_device_clear_perm_entry(struct idxd_device *idxd, + struct idxd_irq_entry *ie) +{ + iowrite32(0, idxd->reg_base + idxd->msix_perm_offset + ie->id * 8); +} + +void idxd_wq_free_irq(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct idxd_irq_entry *ie = &wq->ie; + + synchronize_irq(ie->vector); + free_irq(ie->vector, ie); + idxd_flush_pending_descs(ie); + if (idxd->request_int_handles) + idxd_device_release_int_handle(idxd, ie->int_handle, IDXD_IRQ_MSIX); + idxd_device_clear_perm_entry(idxd, ie); + ie->vector = -1; + ie->int_handle = INVALID_INT_HANDLE; + ie->pasid = INVALID_IOASID; +} + +int idxd_wq_request_irq(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct pci_dev *pdev = idxd->pdev; + struct device *dev = &pdev->dev; + struct idxd_irq_entry *ie; + int rc; + + ie = &wq->ie; + ie->vector = pci_irq_vector(pdev, ie->id); + ie->pasid = device_pasid_enabled(idxd) ? idxd->pasid : INVALID_IOASID; + idxd_device_set_perm_entry(idxd, ie); + + rc = request_threaded_irq(ie->vector, NULL, idxd_wq_thread, 0, "idxd-portal", ie); + if (rc < 0) { + dev_err(dev, "Failed to request irq %d.\n", ie->vector); + goto err_irq; + } + + if (idxd->request_int_handles) { + rc = idxd_device_request_int_handle(idxd, ie->id, &ie->int_handle, + IDXD_IRQ_MSIX); + if (rc < 0) + goto err_int_handle; + } else { + ie->int_handle = ie->id; + } + + return 0; + +err_int_handle: + ie->int_handle = INVALID_INT_HANDLE; + free_irq(ie->vector, ie); +err_irq: + idxd_device_clear_perm_entry(idxd, ie); + ie->pasid = INVALID_IOASID; + return rc; +} + int __drv_enable_wq(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index 2ce873994e33..bfff59617d04 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -289,6 +289,14 @@ static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev) mutex_lock(&wq->wq_lock); wq->type = IDXD_WQT_KERNEL; + + rc = idxd_wq_request_irq(wq); + if (rc < 0) { + idxd->cmd_status = IDXD_SCMD_WQ_IRQ_ERR; + dev_dbg(dev, "WQ %d irq setup failed: %d\n", wq->id, rc); + goto err_irq; + } + rc = __drv_enable_wq(wq); if (rc < 0) { dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc); @@ -329,6 +337,8 @@ err_ref: err_res_alloc: __drv_disable_wq(wq); err: + idxd_wq_free_irq(wq); +err_irq: wq->type = IDXD_WQT_NONE; mutex_unlock(&wq->wq_lock); return rc; @@ -344,6 +354,8 @@ static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev) idxd_wq_free_resources(wq); __drv_disable_wq(wq); percpu_ref_exit(&wq->wq_active); + idxd_wq_free_irq(wq); + wq->type = IDXD_WQT_NONE; mutex_unlock(&wq->wq_lock); } diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index d77be03dd8b0..6353e762286d 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -548,15 +548,10 @@ void idxd_wqs_quiesce(struct idxd_device *idxd); bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc); /* device interrupt control */ -void idxd_msix_perm_setup(struct idxd_device *idxd); -void idxd_msix_perm_clear(struct idxd_device *idxd); irqreturn_t idxd_misc_thread(int vec, void *data); irqreturn_t idxd_wq_thread(int irq, void *data); void idxd_mask_error_interrupts(struct idxd_device *idxd); void idxd_unmask_error_interrupts(struct idxd_device *idxd); -void idxd_mask_msix_vectors(struct idxd_device *idxd); -void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id); -void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id); /* device control */ int idxd_register_idxd_drv(void); @@ -595,6 +590,8 @@ int idxd_wq_disable_pasid(struct idxd_wq *wq); void __idxd_wq_quiesce(struct idxd_wq *wq); void idxd_wq_quiesce(struct idxd_wq *wq); int idxd_wq_init_percpu_ref(struct idxd_wq *wq); +void idxd_wq_free_irq(struct idxd_wq *wq); +int idxd_wq_request_irq(struct idxd_wq *wq); /* submission */ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 03c735727f68..3505efb7ae71 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -90,7 +90,6 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) } dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); - idxd_msix_perm_setup(idxd); ie = idxd_get_ie(idxd, 0); ie->vector = pci_irq_vector(pdev, 0); @@ -99,65 +98,26 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) dev_err(dev, "Failed to allocate misc interrupt.\n"); goto err_misc_irq; } - - dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", ie->vector); + dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector); for (i = 0; i < idxd->max_wqs; i++) { int msix_idx = i + 1; ie = idxd_get_ie(idxd, msix_idx); - - /* MSIX vector 0 special, wq irq entry starts at 1 */ ie->id = msix_idx; - ie->vector = pci_irq_vector(pdev, msix_idx); ie->int_handle = INVALID_INT_HANDLE; - if (device_pasid_enabled(idxd) && i > 0) - ie->pasid = idxd->pasid; - else - ie->pasid = INVALID_IOASID; + ie->pasid = INVALID_IOASID; + spin_lock_init(&ie->list_lock); init_llist_head(&ie->pending_llist); INIT_LIST_HEAD(&ie->work_list); - - rc = request_threaded_irq(ie->vector, NULL, idxd_wq_thread, 0, "idxd-portal", ie); - if (rc < 0) { - dev_err(dev, "Failed to allocate irq %d.\n", ie->vector); - goto err_wq_irqs; - } - - dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", i, ie->vector); - if (idxd->request_int_handles) { - rc = idxd_device_request_int_handle(idxd, i, &ie->int_handle, - IDXD_IRQ_MSIX); - if (rc < 0) { - free_irq(ie->vector, ie); - goto err_wq_irqs; - } - dev_dbg(dev, "int handle requested: %u\n", ie->int_handle); - } else { - ie->int_handle = msix_idx; - } - } idxd_unmask_error_interrupts(idxd); return 0; - err_wq_irqs: - while (--i >= 0) { - ie = &idxd->wqs[i]->ie; - free_irq(ie->vector, ie); - if (ie->int_handle != INVALID_INT_HANDLE) { - idxd_device_release_int_handle(idxd, ie->int_handle, IDXD_IRQ_MSIX); - ie->int_handle = INVALID_INT_HANDLE; - ie->pasid = INVALID_IOASID; - } - ie->vector = -1; - } err_misc_irq: - /* Disable error interrupt generation */ idxd_mask_error_interrupts(idxd); - idxd_msix_perm_clear(idxd); pci_free_irq_vectors(pdev); dev_err(dev, "No usable interrupts\n"); return rc; @@ -167,20 +127,15 @@ static void idxd_cleanup_interrupts(struct idxd_device *idxd) { struct pci_dev *pdev = idxd->pdev; struct idxd_irq_entry *ie; - int i; + int msixcnt; - for (i = 0; i < idxd->irq_cnt; i++) { - ie = idxd_get_ie(idxd, i); - if (ie->int_handle != INVALID_INT_HANDLE) { - idxd_device_release_int_handle(idxd, ie->int_handle, IDXD_IRQ_MSIX); - ie->int_handle = INVALID_INT_HANDLE; - ie->pasid = INVALID_IOASID; - } - free_irq(ie->vector, ie); - ie->vector = -1; - } + msixcnt = pci_msix_vec_count(pdev); + if (msixcnt <= 0) + return; + ie = idxd_get_ie(idxd, 0); idxd_mask_error_interrupts(idxd); + free_irq(ie->vector, ie); pci_free_irq_vectors(pdev); } @@ -592,8 +547,6 @@ static int idxd_probe(struct idxd_device *idxd) if (rc) goto err_config; - dev_dbg(dev, "IDXD interrupt setup complete.\n"); - idxd->major = idxd_cdev_get_major(idxd); rc = perfmon_pmu_init(idxd); @@ -689,31 +642,6 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) return rc; } -static void idxd_flush_pending_descs(struct idxd_irq_entry *ie) -{ - struct idxd_desc *desc, *itr; - struct llist_node *head; - LIST_HEAD(flist); - enum idxd_complete_type ctype; - - spin_lock(&ie->list_lock); - head = llist_del_all(&ie->pending_llist); - if (head) { - llist_for_each_entry_safe(desc, itr, head, llnode) - list_add_tail(&desc->list, &ie->work_list); - } - - list_for_each_entry_safe(desc, itr, &ie->work_list, list) - list_move_tail(&desc->list, &flist); - spin_unlock(&ie->list_lock); - - list_for_each_entry_safe(desc, itr, &flist, list) { - list_del(&desc->list); - ctype = desc->completion->status ? IDXD_COMPLETE_NORMAL : IDXD_COMPLETE_ABORT; - idxd_dma_complete_txd(desc, ctype, true); - } -} - void idxd_wqs_quiesce(struct idxd_device *idxd) { struct idxd_wq *wq; @@ -726,46 +654,19 @@ void idxd_wqs_quiesce(struct idxd_device *idxd) } } -static void idxd_release_int_handles(struct idxd_device *idxd) -{ - struct device *dev = &idxd->pdev->dev; - int i, rc; - - for (i = 1; i < idxd->irq_cnt; i++) { - struct idxd_irq_entry *ie = idxd_get_ie(idxd, i); - - if (ie->int_handle != INVALID_INT_HANDLE) { - rc = idxd_device_release_int_handle(idxd, ie->int_handle, IDXD_IRQ_MSIX); - if (rc < 0) - dev_warn(dev, "irq handle %d release failed\n", ie->int_handle); - else - dev_dbg(dev, "int handle released: %u\n", ie->int_handle); - } - } -} - static void idxd_shutdown(struct pci_dev *pdev) { struct idxd_device *idxd = pci_get_drvdata(pdev); - int rc, i; struct idxd_irq_entry *irq_entry; - int msixcnt = pci_msix_vec_count(pdev); + int rc; rc = idxd_device_disable(idxd); if (rc) dev_err(&pdev->dev, "Disabling device failed\n"); - dev_dbg(&pdev->dev, "%s called\n", __func__); - idxd_mask_msix_vectors(idxd); + irq_entry = &idxd->ie; + synchronize_irq(irq_entry->vector); idxd_mask_error_interrupts(idxd); - - for (i = 0; i < msixcnt; i++) { - irq_entry = idxd_get_ie(idxd, i); - synchronize_irq(irq_entry->vector); - if (i == 0) - continue; - idxd_flush_pending_descs(irq_entry); - } flush_workqueue(idxd->wq); } @@ -773,8 +674,6 @@ static void idxd_remove(struct pci_dev *pdev) { struct idxd_device *idxd = pci_get_drvdata(pdev); struct idxd_irq_entry *irq_entry; - int msixcnt = pci_msix_vec_count(pdev); - int i; idxd_unregister_devices(idxd); /* @@ -790,12 +689,8 @@ static void idxd_remove(struct pci_dev *pdev) if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); - for (i = 0; i < msixcnt; i++) { - irq_entry = idxd_get_ie(idxd, i); - free_irq(irq_entry->vector, irq_entry); - } - idxd_msix_perm_clear(idxd); - idxd_release_int_handles(idxd); + irq_entry = idxd_get_ie(idxd, 0); + free_irq(irq_entry->vector, irq_entry); pci_free_irq_vectors(pdev); pci_iounmap(pdev, idxd->reg_base); iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index a1316f341dd6..743ead5ebc57 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -158,6 +158,9 @@ static void idxd_int_handle_revoke(struct work_struct *work) struct idxd_irq_entry *ie = idxd_get_ie(idxd, i); struct idxd_wq *wq = ie_to_wq(ie); + if (ie->int_handle == INVALID_INT_HANDLE) + continue; + rc = idxd_device_request_int_handle(idxd, i, &new_handle, IDXD_IRQ_MSIX); if (rc < 0) { dev_warn(dev, "get int handle %d failed: %d\n", i, rc); |