From 5d95f40e62e4f2ed3053e9b178471669736cf636 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 11 Oct 2018 16:56:42 +0100 Subject: iommu: Do physical merging in iommu_map_sg() The original motivation for iommu_map_sg() was to give IOMMU drivers the chance to map an IOVA-contiguous scatterlist as efficiently as they could. It turns out that there isn't really much driver-specific business involved there, so now that the default implementation is mandatory let's just improve that - the main thing we're after is to use larger pages wherever possible, and as long as domain->pgsize_bitmap reflects reality, iommu_map() can already do that in a generic way. All we need to do is detect physically-contiguous segments and batch them into a single map operation, since whatever we do here is transparent to our caller and not bound by any segment-length restrictions on the list itself. Speaking of efficiency, there's really very little point in duplicating the checks that iommu_map() is going to do anyway, so those get cleared up in the process. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index edbdf5d6962c..f8ec49e0f6c6 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1712,33 +1712,32 @@ EXPORT_SYMBOL_GPL(iommu_unmap_fast); size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova, struct scatterlist *sg, unsigned int nents, int prot) { - struct scatterlist *s; - size_t mapped = 0; - unsigned int i, min_pagesz; + size_t len = 0, mapped = 0; + phys_addr_t start; + unsigned int i = 0; int ret; - if (unlikely(domain->pgsize_bitmap == 0UL)) - return 0; - - min_pagesz = 1 << __ffs(domain->pgsize_bitmap); + while (i <= nents) { + phys_addr_t s_phys = sg_phys(sg); - for_each_sg(sg, s, nents, i) { - phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset; + if (len && s_phys != start + len) { + ret = iommu_map(domain, iova + mapped, start, len, prot); + if (ret) + goto out_err; - /* - * We are mapping on IOMMU page boundaries, so offset within - * the page must be 0. However, the IOMMU may support pages - * smaller than PAGE_SIZE, so s->offset may still represent - * an offset of that boundary within the CPU page. - */ - if (!IS_ALIGNED(s->offset, min_pagesz)) - goto out_err; + mapped += len; + len = 0; + } - ret = iommu_map(domain, iova + mapped, phys, s->length, prot); - if (ret) - goto out_err; + if (len) { + len += sg->length; + } else { + len = sg->length; + start = s_phys; + } - mapped += s->length; + if (++i < nents) + sg = sg_next(sg); } return mapped; -- cgit v1.2.3 From b0c32912150565130c61104b7789ba4aa0e5f93d Mon Sep 17 00:00:00 2001 From: Hai Nguyen Pham Date: Wed, 17 Oct 2018 11:13:22 +0200 Subject: iommu/ipmmu-vmsa: Hook up r8a77990 DT matching code Support the R-Car E3 (r8a77990) IPMMU. Signed-off-by: Hai Nguyen Pham Signed-off-by: Kazuya Mizuguchi [simon: rebased; dropped no longer required IOMMU_OF_DECLARE hunk] Signed-off-by: Simon Horman Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index b98a03189580..e0adf5d6c13c 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -762,6 +762,7 @@ static const struct soc_device_attribute soc_rcar_gen3[] = { { .soc_id = "r8a7796", }, { .soc_id = "r8a77965", }, { .soc_id = "r8a77970", }, + { .soc_id = "r8a77990", }, { .soc_id = "r8a77995", }, { /* sentinel */ } }; @@ -949,6 +950,9 @@ static const struct of_device_id ipmmu_of_ids[] = { }, { .compatible = "renesas,ipmmu-r8a77970", .data = &ipmmu_features_rcar_gen3, + }, { + .compatible = "renesas,ipmmu-r8a77990", + .data = &ipmmu_features_rcar_gen3, }, { .compatible = "renesas,ipmmu-r8a77995", .data = &ipmmu_features_rcar_gen3, -- cgit v1.2.3 From b65f501627fffc0464607244b05e7a2d3677d654 Mon Sep 17 00:00:00 2001 From: Arvind Yadav Date: Thu, 18 Oct 2018 19:13:38 +0800 Subject: iommu/mediatek: Constify iommu_ops iommu_ops are not supposed to change at runtime. Functions 'iommu_device_set_ops' and 'bus_set_iommu' working with const iommu_ops provided by . So mark the non-const structs as const. Signed-off-by: Arvind Yadav Signed-off-by: Yong Wu (Change the title to iommu/mediatek: xx) Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 4 ++-- drivers/iommu/mtk_iommu_v1.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 44bd5b9166bb..d69a9ffa58a4 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -113,7 +113,7 @@ struct mtk_iommu_domain { struct iommu_domain domain; }; -static struct iommu_ops mtk_iommu_ops; +static const struct iommu_ops mtk_iommu_ops; static LIST_HEAD(m4ulist); /* List all the M4U HWs */ @@ -488,7 +488,7 @@ static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) return iommu_fwspec_add_ids(dev, args->args, 1); } -static struct iommu_ops mtk_iommu_ops = { +static const struct iommu_ops mtk_iommu_ops = { .domain_alloc = mtk_iommu_domain_alloc, .domain_free = mtk_iommu_domain_free, .attach_dev = mtk_iommu_attach_device, diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 0e780848f59b..27867b862d7a 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -362,7 +362,7 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, return pa; } -static struct iommu_ops mtk_iommu_ops; +static const struct iommu_ops mtk_iommu_ops; /* * MTK generation one iommu HW only support one iommu domain, and all the client @@ -524,7 +524,7 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) return 0; } -static struct iommu_ops mtk_iommu_ops = { +static const struct iommu_ops mtk_iommu_ops = { .domain_alloc = mtk_iommu_domain_alloc, .domain_free = mtk_iommu_domain_free, .attach_dev = mtk_iommu_attach_device, -- cgit v1.2.3 From 60fb0083c9d43b6fc7b0e8a25905a2355907c686 Mon Sep 17 00:00:00 2001 From: Fabrizio Castro Date: Thu, 23 Aug 2018 16:33:04 +0100 Subject: iommu/ipmmu-vmsa: Hook up R8A774A1 DT maching code Add support for RZ/G2M (R8A774A1) SoC IPMMUs. Signed-off-by: Fabrizio Castro Reviewed-by: Biju Das Reviewed-by: Simon Horman Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index e0adf5d6c13c..aa0de59f3679 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -758,6 +758,7 @@ static bool ipmmu_slave_whitelist(struct device *dev) } static const struct soc_device_attribute soc_rcar_gen3[] = { + { .soc_id = "r8a774a1", }, { .soc_id = "r8a7795", }, { .soc_id = "r8a7796", }, { .soc_id = "r8a77965", }, @@ -938,6 +939,9 @@ static const struct of_device_id ipmmu_of_ids[] = { { .compatible = "renesas,ipmmu-vmsa", .data = &ipmmu_features_default, + }, { + .compatible = "renesas,ipmmu-r8a774a1", + .data = &ipmmu_features_rcar_gen3, }, { .compatible = "renesas,ipmmu-r8a7795", .data = &ipmmu_features_rcar_gen3, -- cgit v1.2.3 From 31fc9763f9d752b75711156ef6a066f6ea264be5 Mon Sep 17 00:00:00 2001 From: Fabrizio Castro Date: Fri, 17 Aug 2018 15:31:05 +0100 Subject: dt-bindings: iommu: ipmmu-vmsa: Add r8a774a1 support Document RZ/G2M (R8A774A1) SoC bindings. Signed-off-by: Fabrizio Castro Reviewed-by: Biju Das Reviewed-by: Rob Herring Reviewed-by: Simon Horman Signed-off-by: Joerg Roedel --- Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt index 377ee639d103..e285c8ad1b39 100644 --- a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt +++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt @@ -14,6 +14,7 @@ Required Properties: - "renesas,ipmmu-r8a7743" for the R8A7743 (RZ/G1M) IPMMU. - "renesas,ipmmu-r8a7744" for the R8A7744 (RZ/G1N) IPMMU. - "renesas,ipmmu-r8a7745" for the R8A7745 (RZ/G1E) IPMMU. + - "renesas,ipmmu-r8a774a1" for the R8A774A1 (RZ/G2M) IPMMU. - "renesas,ipmmu-r8a7790" for the R8A7790 (R-Car H2) IPMMU. - "renesas,ipmmu-r8a7791" for the R8A7791 (R-Car M2-W) IPMMU. - "renesas,ipmmu-r8a7793" for the R8A7793 (R-Car M2-N) IPMMU. -- cgit v1.2.3 From daedaa33d9c578220b311fbad3748d3ecd5a8f66 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 12 Nov 2018 14:40:08 +0800 Subject: iommu/vtd: Cleanup dma_remapping.h header Commit e61d98d8dad00 ("x64, x2apic/intr-remap: Intel vt-d, IOMMU code reorganization") moved dma_remapping.h from drivers/pci/ to current place. It is entirely VT-d specific, but uses a generic name. This merges dma_remapping.h with include/linux/intel-iommu.h and removes dma_remapping.h as the result. Cc: Ashok Raj Cc: Jacob Pan Cc: Sohil Mehta Suggested-by: Christoph Hellwig Signed-off-by: Lu Baolu Reviewed-by: Christoph Hellwig Reviewed-by: Liu, Yi L Signed-off-by: Joerg Roedel --- arch/x86/kernel/tboot.c | 2 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 2 +- drivers/gpu/drm/i915/intel_display.c | 2 +- drivers/gpu/drm/vmwgfx/vmwgfx_drv.c | 2 +- drivers/misc/mic/scif/scif_rma.c | 2 +- drivers/misc/mic/scif/scif_rma.h | 2 +- include/linux/dma_remapping.h | 58 ------------------------------ include/linux/intel-iommu.h | 49 +++++++++++++++++++++++-- 8 files changed, 53 insertions(+), 66 deletions(-) delete mode 100644 include/linux/dma_remapping.h diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c index a2486f444073..6e5ef8fb8a02 100644 --- a/arch/x86/kernel/tboot.c +++ b/arch/x86/kernel/tboot.c @@ -19,7 +19,7 @@ * */ -#include +#include #include #include #include diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index 1aaccbe7e1de..1c5d04f002bc 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -26,7 +26,7 @@ * */ -#include +#include #include #include #include diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 23d8008a93bb..389d6618c2d5 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -47,7 +47,7 @@ #include #include #include -#include +#include #include /* Primary plane formats for gen <= 3 */ diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c index 61a84b958d67..c3e80a3b09fc 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c @@ -34,7 +34,7 @@ #include #include #include -#include +#include #define VMWGFX_DRIVER_DESC "Linux drm driver for VMware graphics devices" #define VMWGFX_CHIP_SVGAII 0 diff --git a/drivers/misc/mic/scif/scif_rma.c b/drivers/misc/mic/scif/scif_rma.c index c824329f7012..b441f6b0c743 100644 --- a/drivers/misc/mic/scif/scif_rma.c +++ b/drivers/misc/mic/scif/scif_rma.c @@ -15,7 +15,7 @@ * Intel SCIF driver. * */ -#include +#include #include #include #include diff --git a/drivers/misc/mic/scif/scif_rma.h b/drivers/misc/mic/scif/scif_rma.h index fa6722279196..d90a06d4e93b 100644 --- a/drivers/misc/mic/scif/scif_rma.h +++ b/drivers/misc/mic/scif/scif_rma.h @@ -53,7 +53,7 @@ #ifndef SCIF_RMA_H #define SCIF_RMA_H -#include +#include #include #include "../bus/scif_bus.h" diff --git a/include/linux/dma_remapping.h b/include/linux/dma_remapping.h deleted file mode 100644 index 21b3e7d33d68..000000000000 --- a/include/linux/dma_remapping.h +++ /dev/null @@ -1,58 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _DMA_REMAPPING_H -#define _DMA_REMAPPING_H - -/* - * VT-d hardware uses 4KiB page size regardless of host page size. - */ -#define VTD_PAGE_SHIFT (12) -#define VTD_PAGE_SIZE (1UL << VTD_PAGE_SHIFT) -#define VTD_PAGE_MASK (((u64)-1) << VTD_PAGE_SHIFT) -#define VTD_PAGE_ALIGN(addr) (((addr) + VTD_PAGE_SIZE - 1) & VTD_PAGE_MASK) - -#define VTD_STRIDE_SHIFT (9) -#define VTD_STRIDE_MASK (((u64)-1) << VTD_STRIDE_SHIFT) - -#define DMA_PTE_READ (1) -#define DMA_PTE_WRITE (2) -#define DMA_PTE_LARGE_PAGE (1 << 7) -#define DMA_PTE_SNP (1 << 11) - -#define CONTEXT_TT_MULTI_LEVEL 0 -#define CONTEXT_TT_DEV_IOTLB 1 -#define CONTEXT_TT_PASS_THROUGH 2 -/* Extended context entry types */ -#define CONTEXT_TT_PT_PASID 4 -#define CONTEXT_TT_PT_PASID_DEV_IOTLB 5 -#define CONTEXT_TT_MASK (7ULL << 2) - -#define CONTEXT_DINVE (1ULL << 8) -#define CONTEXT_PRS (1ULL << 9) -#define CONTEXT_PASIDE (1ULL << 11) - -struct intel_iommu; -struct dmar_domain; -struct root_entry; - - -#ifdef CONFIG_INTEL_IOMMU -extern int iommu_calculate_agaw(struct intel_iommu *iommu); -extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu); -extern int dmar_disabled; -extern int intel_iommu_enabled; -extern int intel_iommu_tboot_noforce; -#else -static inline int iommu_calculate_agaw(struct intel_iommu *iommu) -{ - return 0; -} -static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu) -{ - return 0; -} -#define dmar_disabled (1) -#define intel_iommu_enabled (0) -#endif - - -#endif diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index b0ae25837361..a58bc05d6798 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -37,9 +36,36 @@ #include /* - * Intel IOMMU register specification per version 1.0 public spec. + * VT-d hardware uses 4KiB page size regardless of host page size. */ +#define VTD_PAGE_SHIFT (12) +#define VTD_PAGE_SIZE (1UL << VTD_PAGE_SHIFT) +#define VTD_PAGE_MASK (((u64)-1) << VTD_PAGE_SHIFT) +#define VTD_PAGE_ALIGN(addr) (((addr) + VTD_PAGE_SIZE - 1) & VTD_PAGE_MASK) + +#define VTD_STRIDE_SHIFT (9) +#define VTD_STRIDE_MASK (((u64)-1) << VTD_STRIDE_SHIFT) + +#define DMA_PTE_READ (1) +#define DMA_PTE_WRITE (2) +#define DMA_PTE_LARGE_PAGE (1 << 7) +#define DMA_PTE_SNP (1 << 11) + +#define CONTEXT_TT_MULTI_LEVEL 0 +#define CONTEXT_TT_DEV_IOTLB 1 +#define CONTEXT_TT_PASS_THROUGH 2 +/* Extended context entry types */ +#define CONTEXT_TT_PT_PASID 4 +#define CONTEXT_TT_PT_PASID_DEV_IOTLB 5 +#define CONTEXT_TT_MASK (7ULL << 2) + +#define CONTEXT_DINVE (1ULL << 8) +#define CONTEXT_PRS (1ULL << 9) +#define CONTEXT_PASIDE (1ULL << 11) +/* + * Intel IOMMU register specification per version 1.0 public spec. + */ #define DMAR_VER_REG 0x0 /* Arch version supported by this IOMMU */ #define DMAR_CAP_REG 0x8 /* Hardware supported capabilities */ #define DMAR_ECAP_REG 0x10 /* Extended capabilities supported */ @@ -632,4 +658,23 @@ bool context_present(struct context_entry *context); struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus, u8 devfn, int alloc); +#ifdef CONFIG_INTEL_IOMMU +extern int iommu_calculate_agaw(struct intel_iommu *iommu); +extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu); +extern int dmar_disabled; +extern int intel_iommu_enabled; +extern int intel_iommu_tboot_noforce; +#else +static inline int iommu_calculate_agaw(struct intel_iommu *iommu) +{ + return 0; +} +static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu) +{ + return 0; +} +#define dmar_disabled (1) +#define intel_iommu_enabled (0) +#endif + #endif -- cgit v1.2.3 From ac3a7092ebcb011759cba39a259f7193e4f6ba62 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 9 Nov 2018 12:07:06 +0100 Subject: iommu/amd: Collect page-table pages in freelist Collect all pages that belong to a page-table in a list and free them after the tree has been traversed. This allows to implement safer page-table updates in subsequent patches. Also move the functions for page-table freeing a bit upwards in the file so that they are usable from the iommu_map() path. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 144 ++++++++++++++++++++++++++-------------------- 1 file changed, 83 insertions(+), 61 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 1167ff0416cf..2655bd91af93 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1317,6 +1317,89 @@ static void domain_flush_devices(struct protection_domain *domain) * ****************************************************************************/ +static void free_page_list(struct page *freelist) +{ + while (freelist != NULL) { + unsigned long p = (unsigned long)page_address(freelist); + freelist = freelist->freelist; + free_page(p); + } +} + +static struct page *free_pt_page(unsigned long pt, struct page *freelist) +{ + struct page *p = virt_to_page((void *)pt); + + p->freelist = freelist; + + return p; +} + +#define DEFINE_FREE_PT_FN(LVL, FN) \ +static struct page *free_pt_##LVL (unsigned long __pt, struct page *freelist) \ +{ \ + unsigned long p; \ + u64 *pt; \ + int i; \ + \ + pt = (u64 *)__pt; \ + \ + for (i = 0; i < 512; ++i) { \ + /* PTE present? */ \ + if (!IOMMU_PTE_PRESENT(pt[i])) \ + continue; \ + \ + /* Large PTE? */ \ + if (PM_PTE_LEVEL(pt[i]) == 0 || \ + PM_PTE_LEVEL(pt[i]) == 7) \ + continue; \ + \ + p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \ + freelist = FN(p, freelist); \ + } \ + \ + return free_pt_page((unsigned long)pt, freelist); \ +} + +DEFINE_FREE_PT_FN(l2, free_pt_page) +DEFINE_FREE_PT_FN(l3, free_pt_l2) +DEFINE_FREE_PT_FN(l4, free_pt_l3) +DEFINE_FREE_PT_FN(l5, free_pt_l4) +DEFINE_FREE_PT_FN(l6, free_pt_l5) + +static void free_pagetable(struct protection_domain *domain) +{ + unsigned long root = (unsigned long)domain->pt_root; + struct page *freelist = NULL; + + switch (domain->mode) { + case PAGE_MODE_NONE: + break; + case PAGE_MODE_1_LEVEL: + freelist = free_pt_page(root, freelist); + break; + case PAGE_MODE_2_LEVEL: + freelist = free_pt_l2(root, freelist); + break; + case PAGE_MODE_3_LEVEL: + freelist = free_pt_l3(root, freelist); + break; + case PAGE_MODE_4_LEVEL: + freelist = free_pt_l4(root, freelist); + break; + case PAGE_MODE_5_LEVEL: + freelist = free_pt_l5(root, freelist); + break; + case PAGE_MODE_6_LEVEL: + freelist = free_pt_l6(root, freelist); + break; + default: + BUG(); + } + + free_page_list(freelist); +} + /* * This function is used to add another level to an IO page table. Adding * another level increases the size of the address space by 9 bits to a size up @@ -1638,67 +1721,6 @@ static void domain_id_free(int id) spin_unlock(&pd_bitmap_lock); } -#define DEFINE_FREE_PT_FN(LVL, FN) \ -static void free_pt_##LVL (unsigned long __pt) \ -{ \ - unsigned long p; \ - u64 *pt; \ - int i; \ - \ - pt = (u64 *)__pt; \ - \ - for (i = 0; i < 512; ++i) { \ - /* PTE present? */ \ - if (!IOMMU_PTE_PRESENT(pt[i])) \ - continue; \ - \ - /* Large PTE? */ \ - if (PM_PTE_LEVEL(pt[i]) == 0 || \ - PM_PTE_LEVEL(pt[i]) == 7) \ - continue; \ - \ - p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \ - FN(p); \ - } \ - free_page((unsigned long)pt); \ -} - -DEFINE_FREE_PT_FN(l2, free_page) -DEFINE_FREE_PT_FN(l3, free_pt_l2) -DEFINE_FREE_PT_FN(l4, free_pt_l3) -DEFINE_FREE_PT_FN(l5, free_pt_l4) -DEFINE_FREE_PT_FN(l6, free_pt_l5) - -static void free_pagetable(struct protection_domain *domain) -{ - unsigned long root = (unsigned long)domain->pt_root; - - switch (domain->mode) { - case PAGE_MODE_NONE: - break; - case PAGE_MODE_1_LEVEL: - free_page(root); - break; - case PAGE_MODE_2_LEVEL: - free_pt_l2(root); - break; - case PAGE_MODE_3_LEVEL: - free_pt_l3(root); - break; - case PAGE_MODE_4_LEVEL: - free_pt_l4(root); - break; - case PAGE_MODE_5_LEVEL: - free_pt_l5(root); - break; - case PAGE_MODE_6_LEVEL: - free_pt_l6(root); - break; - default: - BUG(); - } -} - static void free_gcr3_tbl_level1(u64 *tbl) { u64 *ptr; -- cgit v1.2.3 From 409afa44f9ba1443ef165a9e6d6a9cb5ad6ad176 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 9 Nov 2018 12:07:07 +0100 Subject: iommu/amd: Introduce free_sub_pt() function The function is a more generic version of free_pagetable() and will be used to free only specific sub-trees of a page-table. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 2655bd91af93..1186571f77e1 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1367,12 +1367,10 @@ DEFINE_FREE_PT_FN(l4, free_pt_l3) DEFINE_FREE_PT_FN(l5, free_pt_l4) DEFINE_FREE_PT_FN(l6, free_pt_l5) -static void free_pagetable(struct protection_domain *domain) +static struct page *free_sub_pt(unsigned long root, int mode, + struct page *freelist) { - unsigned long root = (unsigned long)domain->pt_root; - struct page *freelist = NULL; - - switch (domain->mode) { + switch (mode) { case PAGE_MODE_NONE: break; case PAGE_MODE_1_LEVEL: @@ -1397,6 +1395,16 @@ static void free_pagetable(struct protection_domain *domain) BUG(); } + return freelist; +} + +static void free_pagetable(struct protection_domain *domain) +{ + unsigned long root = (unsigned long)domain->pt_root; + struct page *freelist = NULL; + + free_sub_pt(root, domain->mode, freelist); + free_page_list(freelist); } -- cgit v1.2.3 From 69be88520f9dc56d3ec29d7b7d988f55fd248008 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 9 Nov 2018 12:07:08 +0100 Subject: iommu/amd: Ignore page-mode 7 in free_sub_pt() The page-mode 7 is a special one as it marks a final PTE to a page with an intermediary size. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 4 ++++ drivers/iommu/amd_iommu_types.h | 1 + 2 files changed, 5 insertions(+) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 1186571f77e1..49b5d3115e56 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1372,6 +1372,7 @@ static struct page *free_sub_pt(unsigned long root, int mode, { switch (mode) { case PAGE_MODE_NONE: + case PAGE_MODE_7_LEVEL: break; case PAGE_MODE_1_LEVEL: freelist = free_pt_page(root, freelist); @@ -1403,6 +1404,9 @@ static void free_pagetable(struct protection_domain *domain) unsigned long root = (unsigned long)domain->pt_root; struct page *freelist = NULL; + BUG_ON(domain->mode < PAGE_MODE_NONE || + domain->mode > PAGE_MODE_6_LEVEL); + free_sub_pt(root, domain->mode, freelist); free_page_list(freelist); diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index e2b342e65a7b..eae0741f72dc 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -269,6 +269,7 @@ #define PAGE_MODE_4_LEVEL 0x04 #define PAGE_MODE_5_LEVEL 0x05 #define PAGE_MODE_6_LEVEL 0x06 +#define PAGE_MODE_7_LEVEL 0x07 #define PM_LEVEL_SHIFT(x) (12 + ((x) * 9)) #define PM_LEVEL_SIZE(x) (((x) < 6) ? \ -- cgit v1.2.3 From 6d568ef9a622eaaea549b305b7af9c4d91566c84 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 9 Nov 2018 12:07:09 +0100 Subject: iommu/amd: Allow downgrading page-sizes in alloc_pte() Before this patch it was not possible the downgrade a mapping established with page-mode 7 to a mapping using smaller page-sizes, because the pte_level != level check prevented that. Treat page-mode 7 like a non-present mapping and allow to overwrite it in alloc_pte(). Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 49b5d3115e56..6a88ba9321d1 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1460,10 +1460,13 @@ static u64 *alloc_pte(struct protection_domain *domain, while (level > end_lvl) { u64 __pte, __npte; + int pte_level; - __pte = *pte; + __pte = *pte; + pte_level = PM_PTE_LEVEL(__pte); - if (!IOMMU_PTE_PRESENT(__pte)) { + if (!IOMMU_PTE_PRESENT(__pte) || + pte_level == PAGE_MODE_7_LEVEL) { page = (u64 *)get_zeroed_page(gfp); if (!page) return NULL; @@ -1475,10 +1478,13 @@ static u64 *alloc_pte(struct protection_domain *domain, free_page((unsigned long)page); continue; } + + if (pte_level == PAGE_MODE_7_LEVEL) + domain->updated = true; } /* No level skipping support yet */ - if (PM_PTE_LEVEL(*pte) != level) + if (pte_level != level) return NULL; level -= 1; -- cgit v1.2.3 From 9db034d57a8b3cd0aa135a725849cd55a3b2f991 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 9 Nov 2018 12:07:10 +0100 Subject: iommu/amd: Restart loop if cmpxchg64 succeeded in alloc_pte() This makes sure that __pte always contains the correct value when the pointer to the next page-table level is derived. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 6a88ba9321d1..c539b2a59019 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1474,13 +1474,12 @@ static u64 *alloc_pte(struct protection_domain *domain, __npte = PM_LEVEL_PDE(level, iommu_virt_to_phys(page)); /* pte could have been changed somewhere. */ - if (cmpxchg64(pte, __pte, __npte) != __pte) { + if (cmpxchg64(pte, __pte, __npte) != __pte) free_page((unsigned long)page); - continue; - } - - if (pte_level == PAGE_MODE_7_LEVEL) + else if (pte_level == PAGE_MODE_7_LEVEL) domain->updated = true; + + continue; } /* No level skipping support yet */ @@ -1489,7 +1488,7 @@ static u64 *alloc_pte(struct protection_domain *domain, level -= 1; - pte = IOMMU_PTE_PAGE(*pte); + pte = IOMMU_PTE_PAGE(__pte); if (pte_page && level == end_lvl) *pte_page = pte; -- cgit v1.2.3 From 6f820bb995b657ad69197679b4d626cf226bfd35 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 9 Nov 2018 12:07:11 +0100 Subject: iommu/amd: Allow to upgrade page-size Before this patch the iommu_map_page() function failed when it tried to map a huge-page where smaller mappings existed before. With this change the page-table pages of the old mappings are teared down, so that the huge-page can be mapped. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index c539b2a59019..71797b3d67e5 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1557,6 +1557,25 @@ static u64 *fetch_pte(struct protection_domain *domain, return pte; } +static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist) +{ + unsigned long pt; + int mode; + + while (cmpxchg64(pte, pteval, 0) != pteval) { + pr_warn("AMD-Vi: IOMMU pte changed since we read it\n"); + pteval = *pte; + } + + if (!IOMMU_PTE_PRESENT(pteval)) + return freelist; + + pt = (unsigned long)IOMMU_PTE_PAGE(pteval); + mode = IOMMU_PTE_MODE(pteval); + + return free_sub_pt(pt, mode, freelist); +} + /* * Generic mapping functions. It maps a physical address into a DMA * address space. It allocates the page table pages if necessary. @@ -1571,6 +1590,7 @@ static int iommu_map_page(struct protection_domain *dom, int prot, gfp_t gfp) { + struct page *freelist = NULL; u64 __pte, *pte; int i, count; @@ -1587,8 +1607,10 @@ static int iommu_map_page(struct protection_domain *dom, return -ENOMEM; for (i = 0; i < count; ++i) - if (IOMMU_PTE_PRESENT(pte[i])) - return -EBUSY; + freelist = free_clear_pte(&pte[i], pte[i], freelist); + + if (freelist != NULL) + dom->updated = true; if (count > 1) { __pte = PAGE_SIZE_PTE(__sme_set(phys_addr), page_size); @@ -1606,6 +1628,9 @@ static int iommu_map_page(struct protection_domain *dom, update_domain(dom); + /* Everything flushed out, free pages now */ + free_page_list(freelist); + return 0; } -- cgit v1.2.3 From 7a30423a95a11ac474115f07cba7f6071f75ff81 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 9 Nov 2018 12:07:12 +0100 Subject: vfio/type1: Remove map_try_harder() code path The AMD IOMMU driver can now map a huge-page where smaller mappings existed before, so this code-path is no longer triggered. Acked-by: Alex Williamson Signed-off-by: Joerg Roedel --- drivers/vfio/vfio_iommu_type1.c | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index d9fd3188615d..7651cfb14836 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -978,32 +978,6 @@ unlock: return ret; } -/* - * Turns out AMD IOMMU has a page table bug where it won't map large pages - * to a region that previously mapped smaller pages. This should be fixed - * soon, so this is just a temporary workaround to break mappings down into - * PAGE_SIZE. Better to map smaller pages than nothing. - */ -static int map_try_harder(struct vfio_domain *domain, dma_addr_t iova, - unsigned long pfn, long npage, int prot) -{ - long i; - int ret = 0; - - for (i = 0; i < npage; i++, pfn++, iova += PAGE_SIZE) { - ret = iommu_map(domain->domain, iova, - (phys_addr_t)pfn << PAGE_SHIFT, - PAGE_SIZE, prot | domain->prot); - if (ret) - break; - } - - for (; i < npage && i > 0; i--, iova -= PAGE_SIZE) - iommu_unmap(domain->domain, iova, PAGE_SIZE); - - return ret; -} - static int vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova, unsigned long pfn, long npage, int prot) { @@ -1013,11 +987,8 @@ static int vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova, list_for_each_entry(d, &iommu->domain_list, next) { ret = iommu_map(d->domain, iova, (phys_addr_t)pfn << PAGE_SHIFT, npage << PAGE_SHIFT, prot | d->prot); - if (ret) { - if (ret != -EBUSY || - map_try_harder(d, iova, pfn, npage, prot)) - goto unwind; - } + if (ret) + goto unwind; cond_resched(); } -- cgit v1.2.3 From 3569dd07aaad71920c5ea4da2d5cc9a167c1ffd4 Mon Sep 17 00:00:00 2001 From: Sohil Mehta Date: Wed, 21 Nov 2018 15:29:33 -0800 Subject: iommu/vt-d: Handle domain agaw being less than iommu agaw The Intel IOMMU driver opportunistically skips a few top level page tables from the domain paging directory while programming the IOMMU context entry. However there is an implicit assumption in the code that domain's adjusted guest address width (agaw) would always be greater than IOMMU's agaw. The IOMMU capabilities in an upcoming platform cause the domain's agaw to be lower than IOMMU's agaw. The issue is seen when the IOMMU supports both 4-level and 5-level paging. The domain builds a 4-level page table based on agaw of 2. However the IOMMU's agaw is set as 3 (5-level). In this case the code incorrectly tries to skip page page table levels. This causes the IOMMU driver to avoid programming the context entry. The fix handles this case and programs the context entry accordingly. Fixes: de24e55395698 ("iommu/vt-d: Simplify domain_context_mapping_one") Cc: Cc: Ashok Raj Cc: Jacob Pan Cc: Lu Baolu Reviewed-by: Lu Baolu Reported-by: Ramos Falcon, Ernesto R Tested-by: Ricardo Neri Signed-off-by: Sohil Mehta Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index f3ccf025108b..fdf79baf1d79 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2044,7 +2044,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, * than default. Unnecessary for PT mode. */ if (translation != CONTEXT_TT_PASS_THROUGH) { - for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) { + for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) { ret = -ENOMEM; pgd = phys_to_virt(dma_pte_addr(pgd)); if (!dma_pte_present(pgd)) @@ -2058,7 +2058,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, translation = CONTEXT_TT_MULTI_LEVEL; context_set_address_root(context, virt_to_phys(pgd)); - context_set_address_width(context, iommu->agaw); + context_set_address_width(context, agaw); } else { /* * In pass through mode, AW must be programmed to -- cgit v1.2.3 From a6906a8b0ebfbf6330847ba640c8dde8aca679be Mon Sep 17 00:00:00 2001 From: Yangtao Li Date: Thu, 22 Nov 2018 08:54:43 -0500 Subject: iommu/omap: Remove DEBUG_SEQ_FOPS_RO() Because we already have the DEFINE_SHOW_ATTRIBUTE,there is no need to define such a macro.So remove DEBUG_SEQ_FOPS_RO. Signed-off-by: Yangtao Li Signed-off-by: Joerg Roedel --- drivers/iommu/omap-iommu-debug.c | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/drivers/iommu/omap-iommu-debug.c b/drivers/iommu/omap-iommu-debug.c index 50217548c3b8..4abc0ef522a8 100644 --- a/drivers/iommu/omap-iommu-debug.c +++ b/drivers/iommu/omap-iommu-debug.c @@ -159,7 +159,7 @@ static size_t omap_dump_tlb_entries(struct omap_iommu *obj, struct seq_file *s) return 0; } -static int debug_read_tlb(struct seq_file *s, void *data) +static int tlb_show(struct seq_file *s, void *data) { struct omap_iommu *obj = s->private; @@ -210,7 +210,7 @@ static void dump_ioptable(struct seq_file *s) spin_unlock(&obj->page_table_lock); } -static int debug_read_pagetable(struct seq_file *s, void *data) +static int pagetable_show(struct seq_file *s, void *data) { struct omap_iommu *obj = s->private; @@ -228,35 +228,22 @@ static int debug_read_pagetable(struct seq_file *s, void *data) return 0; } -#define DEBUG_SEQ_FOPS_RO(name) \ - static int debug_open_##name(struct inode *inode, struct file *file) \ - { \ - return single_open(file, debug_read_##name, inode->i_private); \ - } \ - \ - static const struct file_operations debug_##name##_fops = { \ - .open = debug_open_##name, \ - .read = seq_read, \ - .llseek = seq_lseek, \ - .release = single_release, \ - } - #define DEBUG_FOPS_RO(name) \ - static const struct file_operations debug_##name##_fops = { \ + static const struct file_operations name##_fops = { \ .open = simple_open, \ .read = debug_read_##name, \ .llseek = generic_file_llseek, \ } DEBUG_FOPS_RO(regs); -DEBUG_SEQ_FOPS_RO(tlb); -DEBUG_SEQ_FOPS_RO(pagetable); +DEFINE_SHOW_ATTRIBUTE(tlb); +DEFINE_SHOW_ATTRIBUTE(pagetable); #define __DEBUG_ADD_FILE(attr, mode) \ { \ struct dentry *dent; \ dent = debugfs_create_file(#attr, mode, obj->debug_dir, \ - obj, &debug_##attr##_fops); \ + obj, &attr##_fops); \ if (!dent) \ goto err; \ } -- cgit v1.2.3 From 062e52a5af9cce644593fbd115d6db1b4558a8cf Mon Sep 17 00:00:00 2001 From: Yangtao Li Date: Thu, 22 Nov 2018 08:30:47 -0500 Subject: iommu/tegra: Change to use DEFINE_SHOW_ATTRIBUTE macro Use DEFINE_SHOW_ATTRIBUTE macro to simplify the code. Signed-off-by: Yangtao Li Acked-by: Thierry Reding Signed-off-by: Joerg Roedel --- drivers/iommu/tegra-smmu.c | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 0d03341317c4..97b3c0461831 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -926,17 +926,7 @@ static int tegra_smmu_swgroups_show(struct seq_file *s, void *data) return 0; } -static int tegra_smmu_swgroups_open(struct inode *inode, struct file *file) -{ - return single_open(file, tegra_smmu_swgroups_show, inode->i_private); -} - -static const struct file_operations tegra_smmu_swgroups_fops = { - .open = tegra_smmu_swgroups_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(tegra_smmu_swgroups); static int tegra_smmu_clients_show(struct seq_file *s, void *data) { @@ -964,17 +954,7 @@ static int tegra_smmu_clients_show(struct seq_file *s, void *data) return 0; } -static int tegra_smmu_clients_open(struct inode *inode, struct file *file) -{ - return single_open(file, tegra_smmu_clients_show, inode->i_private); -} - -static const struct file_operations tegra_smmu_clients_fops = { - .open = tegra_smmu_clients_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; +DEFINE_SHOW_ATTRIBUTE(tegra_smmu_clients); static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu) { -- cgit v1.2.3 From 101fa0371478aa0457c58e175b8f66a110c4b24c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 27 Nov 2018 16:22:31 +0100 Subject: iommu/amd: Use pr_fmt() Make use of pr_fmt instead of having the 'AMD-Vi' prefix added manually at every printk() call. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 38 +++++++++++++------------ drivers/iommu/amd_iommu_init.c | 64 ++++++++++++++++++++++-------------------- drivers/iommu/amd_iommu_v2.c | 2 ++ 3 files changed, 55 insertions(+), 49 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 71797b3d67e5..81fff9e49a3f 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -17,6 +17,8 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#define pr_fmt(fmt) "AMD-Vi: " fmt + #include #include #include @@ -279,7 +281,7 @@ static u16 get_alias(struct device *dev) return pci_alias; } - pr_info("AMD-Vi: Using IVRS reported alias %02x:%02x.%d " + pr_info("Using IVRS reported alias %02x:%02x.%d " "for device %s[%04x:%04x], kernel reported alias " "%02x:%02x.%d\n", PCI_BUS_NUM(ivrs_alias), PCI_SLOT(ivrs_alias), PCI_FUNC(ivrs_alias), dev_name(dev), pdev->vendor, pdev->device, @@ -293,7 +295,7 @@ static u16 get_alias(struct device *dev) if (pci_alias == devid && PCI_BUS_NUM(ivrs_alias) == pdev->bus->number) { pci_add_dma_alias(pdev, ivrs_alias & 0xff); - pr_info("AMD-Vi: Added PCI DMA alias %02x.%d for %s\n", + pr_info("Added PCI DMA alias %02x.%d for %s\n", PCI_SLOT(ivrs_alias), PCI_FUNC(ivrs_alias), dev_name(dev)); } @@ -513,7 +515,7 @@ static void dump_dte_entry(u16 devid) int i; for (i = 0; i < 4; ++i) - pr_err("AMD-Vi: DTE[%d]: %016llx\n", i, + pr_err("DTE[%d]: %016llx\n", i, amd_iommu_dev_table[devid].data[i]); } @@ -523,7 +525,7 @@ static void dump_command(unsigned long phys_addr) int i; for (i = 0; i < 4; ++i) - pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]); + pr_err("CMD[%d]: %08x\n", i, cmd->data[i]); } static void amd_iommu_report_page_fault(u16 devid, u16 domain_id, @@ -541,7 +543,7 @@ static void amd_iommu_report_page_fault(u16 devid, u16 domain_id, dev_err(&pdev->dev, "AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%016llx flags=0x%04x]\n", domain_id, address, flags); } else if (printk_ratelimit()) { - pr_err("AMD-Vi: Event logged [IO_PAGE_FAULT device=%02x:%02x.%x domain=0x%04x address=0x%016llx flags=0x%04x]\n", + pr_err("Event logged [IO_PAGE_FAULT device=%02x:%02x.%x domain=0x%04x address=0x%016llx flags=0x%04x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), domain_id, address, flags); } @@ -568,7 +570,7 @@ retry: if (type == 0) { /* Did we hit the erratum? */ if (++count == LOOP_TIMEOUT) { - pr_err("AMD-Vi: No event written to event log\n"); + pr_err("No event written to event log\n"); return; } udelay(1); @@ -654,7 +656,7 @@ static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw) struct amd_iommu_fault fault; if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) { - pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n"); + pr_err_ratelimited("Unknown PPR request received\n"); return; } @@ -759,12 +761,12 @@ static void iommu_poll_ga_log(struct amd_iommu *iommu) if (!iommu_ga_log_notifier) break; - pr_debug("AMD-Vi: %s: devid=%#x, ga_tag=%#x\n", + pr_debug("%s: devid=%#x, ga_tag=%#x\n", __func__, GA_DEVID(log_entry), GA_TAG(log_entry)); if (iommu_ga_log_notifier(GA_TAG(log_entry)) != 0) - pr_err("AMD-Vi: GA log notifier failed.\n"); + pr_err("GA log notifier failed.\n"); break; default: break; @@ -789,18 +791,18 @@ irqreturn_t amd_iommu_int_thread(int irq, void *data) iommu->mmio_base + MMIO_STATUS_OFFSET); if (status & MMIO_STATUS_EVT_INT_MASK) { - pr_devel("AMD-Vi: Processing IOMMU Event Log\n"); + pr_devel("Processing IOMMU Event Log\n"); iommu_poll_events(iommu); } if (status & MMIO_STATUS_PPR_INT_MASK) { - pr_devel("AMD-Vi: Processing IOMMU PPR Log\n"); + pr_devel("Processing IOMMU PPR Log\n"); iommu_poll_ppr_log(iommu); } #ifdef CONFIG_IRQ_REMAP if (status & MMIO_STATUS_GALOG_INT_MASK) { - pr_devel("AMD-Vi: Processing IOMMU GA Log\n"); + pr_devel("Processing IOMMU GA Log\n"); iommu_poll_ga_log(iommu); } #endif @@ -844,7 +846,7 @@ static int wait_on_sem(volatile u64 *sem) } if (i == LOOP_TIMEOUT) { - pr_alert("AMD-Vi: Completion-Wait loop timed out\n"); + pr_alert("Completion-Wait loop timed out\n"); return -EIO; } @@ -1036,7 +1038,7 @@ again: /* Skip udelay() the first time around */ if (count++) { if (count == LOOP_TIMEOUT) { - pr_err("AMD-Vi: Command buffer timeout\n"); + pr_err("Command buffer timeout\n"); return -EIO; } @@ -2854,9 +2856,9 @@ int __init amd_iommu_init_dma_ops(void) dma_ops = &dma_direct_ops; if (amd_iommu_unmap_flush) - pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n"); + pr_info("IO/TLB flush on unmap enabled\n"); else - pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n"); + pr_info("Lazy IO/TLB flushing enabled\n"); return 0; @@ -2961,7 +2963,7 @@ static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) case IOMMU_DOMAIN_DMA: dma_domain = dma_ops_domain_alloc(); if (!dma_domain) { - pr_err("AMD-Vi: Failed to allocate\n"); + pr_err("Failed to allocate\n"); return NULL; } pdomain = &dma_domain->domain; @@ -4382,7 +4384,7 @@ static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) * legacy mode. So, we force legacy mode instead. */ if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) { - pr_debug("AMD-Vi: %s: Fall back to using intr legacy remap\n", + pr_debug("%s: Fall back to using intr legacy remap\n", __func__); pi_data->is_guest_mode = false; } diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index bb2cd29e1658..e0a04975ce00 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -17,6 +17,8 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#define pr_fmt(fmt) "AMD-Vi: " fmt + #include #include #include @@ -443,9 +445,9 @@ static void iommu_disable(struct amd_iommu *iommu) static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end) { if (!request_mem_region(address, end, "amd_iommu")) { - pr_err("AMD-Vi: Can not reserve memory region %llx-%llx for mmio\n", + pr_err("Can not reserve memory region %llx-%llx for mmio\n", address, end); - pr_err("AMD-Vi: This is a BIOS bug. Please contact your hardware vendor\n"); + pr_err("This is a BIOS bug. Please contact your hardware vendor\n"); return NULL; } @@ -512,7 +514,7 @@ static int __init find_last_devid_from_ivhd(struct ivhd_header *h) u32 ivhd_size = get_ivhd_header_size(h); if (!ivhd_size) { - pr_err("AMD-Vi: Unsupported IVHD type %#x\n", h->type); + pr_err("Unsupported IVHD type %#x\n", h->type); return -EINVAL; } @@ -553,7 +555,7 @@ static int __init check_ivrs_checksum(struct acpi_table_header *table) checksum += p[i]; if (checksum != 0) { /* ACPI table corrupt */ - pr_err(FW_BUG "AMD-Vi: IVRS invalid checksum\n"); + pr_err(FW_BUG "IVRS invalid checksum\n"); return -ENODEV; } @@ -1027,7 +1029,7 @@ static int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line) if (!(entry->id == id && entry->cmd_line)) continue; - pr_info("AMD-Vi: Command-line override present for %s id %d - ignoring\n", + pr_info("Command-line override present for %s id %d - ignoring\n", type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id); *devid = entry->devid; @@ -1060,7 +1062,7 @@ static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u16 *devid, !entry->cmd_line) continue; - pr_info("AMD-Vi: Command-line override for hid:%s uid:%s\n", + pr_info("Command-line override for hid:%s uid:%s\n", hid, uid); *devid = entry->devid; return 0; @@ -1076,7 +1078,7 @@ static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u16 *devid, entry->cmd_line = cmd_line; entry->root_devid = (entry->devid & (~0x7)); - pr_info("AMD-Vi:%s, add hid:%s, uid:%s, rdevid:%d\n", + pr_info("%s, add hid:%s, uid:%s, rdevid:%d\n", entry->cmd_line ? "cmd" : "ivrs", entry->hid, entry->uid, entry->root_devid); @@ -1172,7 +1174,7 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, */ ivhd_size = get_ivhd_header_size(h); if (!ivhd_size) { - pr_err("AMD-Vi: Unsupported IVHD type %#x\n", h->type); + pr_err("Unsupported IVHD type %#x\n", h->type); return -EINVAL; } @@ -1454,7 +1456,7 @@ static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu) pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8)); pci_write_config_dword(iommu->dev, 0xf4, value | 0x4); - pr_info("AMD-Vi: Applying erratum 746 workaround for IOMMU at %s\n", + pr_info("Applying erratum 746 workaround for IOMMU at %s\n", dev_name(&iommu->dev->dev)); /* Clear the enable writing bit */ @@ -1485,7 +1487,7 @@ static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu) /* Set L2_DEBUG_3[AtsIgnoreIWDis] = 1 */ iommu_write_l2(iommu, 0x47, value | BIT(0)); - pr_info("AMD-Vi: Applying ATS write check workaround for IOMMU at %s\n", + pr_info("Applying ATS write check workaround for IOMMU at %s\n", dev_name(&iommu->dev->dev)); } @@ -1505,7 +1507,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) iommu->index = amd_iommus_present++; if (unlikely(iommu->index >= MAX_IOMMUS)) { - WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n"); + WARN(1, "System has more IOMMUs than supported by this driver\n"); return -ENOSYS; } @@ -1673,12 +1675,12 @@ static void init_iommu_perf_ctr(struct amd_iommu *iommu) if ((iommu_pc_get_set_reg(iommu, 0, 0, 0, &val, true)) || (iommu_pc_get_set_reg(iommu, 0, 0, 0, &val2, false)) || (val != val2)) { - pr_err("AMD-Vi: Unable to write to IOMMU perf counter.\n"); + pr_err("Unable to write to IOMMU perf counter.\n"); amd_iommu_pc_present = false; return; } - pr_info("AMD-Vi: IOMMU performance counters supported\n"); + pr_info("IOMMU performance counters supported\n"); val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET); iommu->max_banks = (u8) ((val >> 12) & 0x3f); @@ -1839,11 +1841,11 @@ static void print_iommu_info(void) for_each_iommu(iommu) { int i; - pr_info("AMD-Vi: Found IOMMU at %s cap 0x%hx\n", + pr_info("Found IOMMU at %s cap 0x%hx\n", dev_name(&iommu->dev->dev), iommu->cap_ptr); if (iommu->cap & (1 << IOMMU_CAP_EFR)) { - pr_info("AMD-Vi: Extended features (%#llx):\n", + pr_info("Extended features (%#llx):\n", iommu->features); for (i = 0; i < ARRAY_SIZE(feat_str); ++i) { if (iommu_feature(iommu, (1ULL << i))) @@ -1857,11 +1859,11 @@ static void print_iommu_info(void) } } if (irq_remapping_enabled) { - pr_info("AMD-Vi: Interrupt remapping enabled\n"); + pr_info("Interrupt remapping enabled\n"); if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) - pr_info("AMD-Vi: virtual APIC enabled\n"); + pr_info("Virtual APIC enabled\n"); if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) - pr_info("AMD-Vi: X2APIC enabled\n"); + pr_info("X2APIC enabled\n"); } } @@ -2375,7 +2377,7 @@ static bool __init check_ioapic_information(void) devid = get_ioapic_devid(id); if (devid < 0) { - pr_err("%sAMD-Vi: IOAPIC[%d] not in IVRS table\n", + pr_err("%s: IOAPIC[%d] not in IVRS table\n", fw_bug, id); ret = false; } else if (devid == IOAPIC_SB_DEVID) { @@ -2393,11 +2395,11 @@ static bool __init check_ioapic_information(void) * when the BIOS is buggy and provides us the wrong * device id for the IOAPIC in the system. */ - pr_err("%sAMD-Vi: No southbridge IOAPIC found\n", fw_bug); + pr_err("%s: No southbridge IOAPIC found\n", fw_bug); } if (!ret) - pr_err("AMD-Vi: Disabling interrupt remapping\n"); + pr_err("Disabling interrupt remapping\n"); return ret; } @@ -2452,7 +2454,7 @@ static int __init early_amd_iommu_init(void) return -ENODEV; else if (ACPI_FAILURE(status)) { const char *err = acpi_format_exception(status); - pr_err("AMD-Vi: IVRS table error: %s\n", err); + pr_err("IVRS table error: %s\n", err); return -EINVAL; } @@ -2605,7 +2607,7 @@ static bool detect_ivrs(void) return false; else if (ACPI_FAILURE(status)) { const char *err = acpi_format_exception(status); - pr_err("AMD-Vi: IVRS table error: %s\n", err); + pr_err("IVRS table error: %s\n", err); return false; } @@ -2640,7 +2642,7 @@ static int __init state_next(void) ret = early_amd_iommu_init(); init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED; if (init_state == IOMMU_ACPI_FINISHED && amd_iommu_disabled) { - pr_info("AMD-Vi: AMD IOMMU disabled on kernel command-line\n"); + pr_info("AMD IOMMU disabled on kernel command-line\n"); free_dma_resources(); free_iommu_resources(); init_state = IOMMU_CMDLINE_DISABLED; @@ -2787,7 +2789,7 @@ static bool amd_iommu_sme_check(void) (boot_cpu_data.microcode <= 0x080011ff)) return true; - pr_notice("AMD-Vi: IOMMU not currently supported when SME is active\n"); + pr_notice("IOMMU not currently supported when SME is active\n"); return false; } @@ -2872,12 +2874,12 @@ static int __init parse_ivrs_ioapic(char *str) ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn); if (ret != 4) { - pr_err("AMD-Vi: Invalid command line: ivrs_ioapic%s\n", str); + pr_err("Invalid command line: ivrs_ioapic%s\n", str); return 1; } if (early_ioapic_map_size == EARLY_MAP_SIZE) { - pr_err("AMD-Vi: Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n", + pr_err("Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n", str); return 1; } @@ -2902,12 +2904,12 @@ static int __init parse_ivrs_hpet(char *str) ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn); if (ret != 4) { - pr_err("AMD-Vi: Invalid command line: ivrs_hpet%s\n", str); + pr_err("Invalid command line: ivrs_hpet%s\n", str); return 1; } if (early_hpet_map_size == EARLY_MAP_SIZE) { - pr_err("AMD-Vi: Early HPET map overflow - ignoring ivrs_hpet%s\n", + pr_err("Early HPET map overflow - ignoring ivrs_hpet%s\n", str); return 1; } @@ -2932,7 +2934,7 @@ static int __init parse_ivrs_acpihid(char *str) ret = sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid); if (ret != 4) { - pr_err("AMD-Vi: Invalid command line: ivrs_acpihid(%s)\n", str); + pr_err("Invalid command line: ivrs_acpihid(%s)\n", str); return 1; } @@ -2941,7 +2943,7 @@ static int __init parse_ivrs_acpihid(char *str) uid = p; if (!hid || !(*hid) || !uid) { - pr_err("AMD-Vi: Invalid command line: hid or uid\n"); + pr_err("Invalid command line: hid or uid\n"); return 1; } diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index fd552235bd13..23dae9348ace 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -16,6 +16,8 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#define pr_fmt(fmt) "AMD-Vi: " fmt + #include #include #include -- cgit v1.2.3 From 1a21ee1aabf7ff9484f2eb122476d80c7f60a620 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 27 Nov 2018 16:43:57 +0100 Subject: iommu/amd: Fix line-break in error log reporting With the switch to dev_err for reporting errors from the iommu log there was an unwanted newline introduced. The reason was that the reporting was done in multiple dev_err() calls, and dev_err adds a newline after every call. Fix it by printing the log messages with only one dev_err() call. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 81fff9e49a3f..42698cf2b666 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -540,7 +540,7 @@ static void amd_iommu_report_page_fault(u16 devid, u16 domain_id, dev_data = get_dev_data(&pdev->dev); if (dev_data && __ratelimit(&dev_data->rs)) { - dev_err(&pdev->dev, "AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%016llx flags=0x%04x]\n", + dev_err(&pdev->dev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%016llx flags=0x%04x]\n", domain_id, address, flags); } else if (printk_ratelimit()) { pr_err("Event logged [IO_PAGE_FAULT device=%02x:%02x.%x domain=0x%04x address=0x%016llx flags=0x%04x]\n", @@ -580,43 +580,41 @@ retry: if (type == EVENT_TYPE_IO_FAULT) { amd_iommu_report_page_fault(devid, pasid, address, flags); return; - } else { - dev_err(dev, "AMD-Vi: Event logged ["); } switch (type) { case EVENT_TYPE_ILL_DEV: - dev_err(dev, "ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x pasid=0x%05x address=0x%016llx flags=0x%04x]\n", + dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x pasid=0x%05x address=0x%016llx flags=0x%04x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), pasid, address, flags); dump_dte_entry(devid); break; case EVENT_TYPE_DEV_TAB_ERR: - dev_err(dev, "DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x " + dev_err(dev, "Event logged [DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x " "address=0x%016llx flags=0x%04x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), address, flags); break; case EVENT_TYPE_PAGE_TAB_ERR: - dev_err(dev, "PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x domain=0x%04x address=0x%016llx flags=0x%04x]\n", + dev_err(dev, "Event logged [PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x domain=0x%04x address=0x%016llx flags=0x%04x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), pasid, address, flags); break; case EVENT_TYPE_ILL_CMD: - dev_err(dev, "ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address); + dev_err(dev, "Event logged [ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address); dump_command(address); break; case EVENT_TYPE_CMD_HARD_ERR: - dev_err(dev, "COMMAND_HARDWARE_ERROR address=0x%016llx flags=0x%04x]\n", + dev_err(dev, "Event logged [COMMAND_HARDWARE_ERROR address=0x%016llx flags=0x%04x]\n", address, flags); break; case EVENT_TYPE_IOTLB_INV_TO: - dev_err(dev, "IOTLB_INV_TIMEOUT device=%02x:%02x.%x address=0x%016llx]\n", + dev_err(dev, "Event logged [IOTLB_INV_TIMEOUT device=%02x:%02x.%x address=0x%016llx]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), address); break; case EVENT_TYPE_INV_DEV_REQ: - dev_err(dev, "INVALID_DEVICE_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%016llx flags=0x%04x]\n", + dev_err(dev, "Event logged [INVALID_DEVICE_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%016llx flags=0x%04x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), pasid, address, flags); break; @@ -624,12 +622,12 @@ retry: pasid = ((event[0] >> 16) & 0xFFFF) | ((event[1] << 6) & 0xF0000); tag = event[1] & 0x03FF; - dev_err(dev, "INVALID_PPR_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%016llx flags=0x%04x]\n", + dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%016llx flags=0x%04x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), pasid, address, flags); break; default: - dev_err(dev, "UNKNOWN event[0]=0x%08x event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n", + dev_err(dev, "Event logged [UNKNOWN event[0]=0x%08x event[1]=0x%08x event[2]=0x%08x event[3]=0x%08x\n", event[0], event[1], event[2], event[3]); } -- cgit v1.2.3 From 6f5086a6e3383ae6a1b08a88836fcfda7d171ca4 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 27 Nov 2018 17:18:52 +0100 Subject: iommu/amd: Remove leading 0s in error log messages Remove the leading 0s in the address field of the error log messages. Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 42698cf2b666..c5d78c87248a 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -540,10 +540,10 @@ static void amd_iommu_report_page_fault(u16 devid, u16 domain_id, dev_data = get_dev_data(&pdev->dev); if (dev_data && __ratelimit(&dev_data->rs)) { - dev_err(&pdev->dev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%016llx flags=0x%04x]\n", + dev_err(&pdev->dev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%llx flags=0x%04x]\n", domain_id, address, flags); } else if (printk_ratelimit()) { - pr_err("Event logged [IO_PAGE_FAULT device=%02x:%02x.%x domain=0x%04x address=0x%016llx flags=0x%04x]\n", + pr_err("Event logged [IO_PAGE_FAULT device=%02x:%02x.%x domain=0x%04x address=0x%llx flags=0x%04x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), domain_id, address, flags); } @@ -584,37 +584,37 @@ retry: switch (type) { case EVENT_TYPE_ILL_DEV: - dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x pasid=0x%05x address=0x%016llx flags=0x%04x]\n", + dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), pasid, address, flags); dump_dte_entry(devid); break; case EVENT_TYPE_DEV_TAB_ERR: dev_err(dev, "Event logged [DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x " - "address=0x%016llx flags=0x%04x]\n", + "address=0x%llx flags=0x%04x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), address, flags); break; case EVENT_TYPE_PAGE_TAB_ERR: - dev_err(dev, "Event logged [PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x domain=0x%04x address=0x%016llx flags=0x%04x]\n", + dev_err(dev, "Event logged [PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x domain=0x%04x address=0x%llx flags=0x%04x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), pasid, address, flags); break; case EVENT_TYPE_ILL_CMD: - dev_err(dev, "Event logged [ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address); + dev_err(dev, "Event logged [ILLEGAL_COMMAND_ERROR address=0x%llx]\n", address); dump_command(address); break; case EVENT_TYPE_CMD_HARD_ERR: - dev_err(dev, "Event logged [COMMAND_HARDWARE_ERROR address=0x%016llx flags=0x%04x]\n", + dev_err(dev, "Event logged [COMMAND_HARDWARE_ERROR address=0x%llx flags=0x%04x]\n", address, flags); break; case EVENT_TYPE_IOTLB_INV_TO: - dev_err(dev, "Event logged [IOTLB_INV_TIMEOUT device=%02x:%02x.%x address=0x%016llx]\n", + dev_err(dev, "Event logged [IOTLB_INV_TIMEOUT device=%02x:%02x.%x address=0x%llx]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), address); break; case EVENT_TYPE_INV_DEV_REQ: - dev_err(dev, "Event logged [INVALID_DEVICE_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%016llx flags=0x%04x]\n", + dev_err(dev, "Event logged [INVALID_DEVICE_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), pasid, address, flags); break; @@ -622,7 +622,7 @@ retry: pasid = ((event[0] >> 16) & 0xFFFF) | ((event[1] << 6) & 0xF0000); tag = event[1] & 0x03FF; - dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%016llx flags=0x%04x]\n", + dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), pasid, address, flags); break; -- cgit v1.2.3 From b7ee92c6ff0e023a473b2500d771f6bf504fcc1a Mon Sep 17 00:00:00 2001 From: Yoshihiro Shimoda Date: Wed, 28 Nov 2018 09:23:36 +0000 Subject: iommu/ipmmu-vmsa: Modify ipmmu_slave_whitelist() to check SoC revisions Some R-Car Gen3 SoCs has hardware restrictions on the IPMMU. So, to check whether this R-Car Gen3 SoC can use the IPMMU correctly, this patch modifies the ipmmu_slave_whitelist(). Signed-off-by: Yoshihiro Shimoda Reviewed-by: Geert Uytterhoeven Reviewed-by: Simon Horman Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index aa0de59f3679..46906a763d69 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -751,12 +751,6 @@ static int ipmmu_init_platform_device(struct device *dev, return 0; } -static bool ipmmu_slave_whitelist(struct device *dev) -{ - /* By default, do not allow use of IPMMU */ - return false; -} - static const struct soc_device_attribute soc_rcar_gen3[] = { { .soc_id = "r8a774a1", }, { .soc_id = "r8a7795", }, @@ -768,11 +762,35 @@ static const struct soc_device_attribute soc_rcar_gen3[] = { { /* sentinel */ } }; +static const struct soc_device_attribute soc_rcar_gen3_whitelist[] = { + { .soc_id = "r8a7795", .revision = "ES3.*" }, + { .soc_id = "r8a77965", }, + { .soc_id = "r8a77990", }, + { .soc_id = "r8a77995", }, + { /* sentinel */ } +}; + +static bool ipmmu_slave_whitelist(struct device *dev) +{ + /* + * For R-Car Gen3 use a white list to opt-in slave devices. + * For Other SoCs, this returns true anyway. + */ + if (!soc_device_match(soc_rcar_gen3)) + return true; + + /* Check whether this R-Car Gen3 can use the IPMMU correctly or not */ + if (!soc_device_match(soc_rcar_gen3_whitelist)) + return false; + + /* By default, do not allow use of IPMMU */ + return false; +} + static int ipmmu_of_xlate(struct device *dev, struct of_phandle_args *spec) { - /* For R-Car Gen3 use a white list to opt-in slave devices */ - if (soc_device_match(soc_rcar_gen3) && !ipmmu_slave_whitelist(dev)) + if (!ipmmu_slave_whitelist(dev)) return -ENODEV; iommu_fwspec_add_ids(dev, spec->args, 1); -- cgit v1.2.3 From 807596491d33d016e420d217cf1350aef2d2b3f7 Mon Sep 17 00:00:00 2001 From: Yoshihiro Shimoda Date: Wed, 28 Nov 2018 09:23:36 +0000 Subject: iommu/ipmmu-vmsa: Add an array of slave devices whitelist To avoid adding copy and pasted strcmp codes in the future, this patch adds an array "rcar_gen3_slave_whitelist" to check whether the device can work with the IPMMU or not. Signed-off-by: Yoshihiro Shimoda Reviewed-by: Geert Uytterhoeven Reviewed-by: Simon Horman Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 46906a763d69..331e86839a8c 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -770,8 +770,13 @@ static const struct soc_device_attribute soc_rcar_gen3_whitelist[] = { { /* sentinel */ } }; +static const char * const rcar_gen3_slave_whitelist[] = { +}; + static bool ipmmu_slave_whitelist(struct device *dev) { + unsigned int i; + /* * For R-Car Gen3 use a white list to opt-in slave devices. * For Other SoCs, this returns true anyway. @@ -783,7 +788,13 @@ static bool ipmmu_slave_whitelist(struct device *dev) if (!soc_device_match(soc_rcar_gen3_whitelist)) return false; - /* By default, do not allow use of IPMMU */ + /* Check whether this slave device can work with the IPMMU */ + for (i = 0; i < ARRAY_SIZE(rcar_gen3_slave_whitelist); i++) { + if (!strcmp(dev_name(dev), rcar_gen3_slave_whitelist[i])) + return true; + } + + /* Otherwise, do not allow use of IPMMU */ return false; } -- cgit v1.2.3 From 753a31a46b1afebf2399c005bb81eccf227393c1 Mon Sep 17 00:00:00 2001 From: Yangtao Li Date: Fri, 30 Nov 2018 09:35:21 -0500 Subject: irq_remapping: Remove unused header files seq_file.h does not need to be included,so remove it. Signed-off-by: Yangtao Li Signed-off-by: Joerg Roedel --- drivers/iommu/irq_remapping.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c index 7d0f3074d41d..b94ebd42edd8 100644 --- a/drivers/iommu/irq_remapping.c +++ b/drivers/iommu/irq_remapping.c @@ -1,4 +1,3 @@ -#include #include #include #include -- cgit v1.2.3 From c1af7b4013e86bcfd4e1e8f0eeb7ae914ca7e819 Mon Sep 17 00:00:00 2001 From: Paul Gortmaker Date: Sat, 1 Dec 2018 14:19:09 -0500 Subject: iommu: Audit and remove any unnecessary uses of module.h Historically a lot of these existed because we did not have a distinction between what was modular code and what was providing support to modules via EXPORT_SYMBOL and friends. That changed when we forked out support for the latter into the export.h file. This means we should be able to reduce the usage of module.h in code that is obj-y Makefile or bool Kconfig. The advantage in removing such instances is that module.h itself sources about 15 other headers; adding significantly to what we feed cpp, and it can obscure what headers we are effectively using. Since module.h might have been the implicit source for init.h (for __init) and for export.h (for EXPORT_SYMBOL) we consider each instance for the presence of either and replace as needed. Cc: Joerg Roedel Cc: iommu@lists.linux-foundation.org Signed-off-by: Paul Gortmaker Signed-off-by: Joerg Roedel --- drivers/iommu/iommu-sysfs.c | 2 +- drivers/iommu/iommu.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/iommu-sysfs.c b/drivers/iommu/iommu-sysfs.c index 36d1a7ce7fc4..c298330ba2b7 100644 --- a/drivers/iommu/iommu-sysfs.c +++ b/drivers/iommu/iommu-sysfs.c @@ -11,7 +11,7 @@ #include #include -#include +#include #include /* diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index f8ec49e0f6c6..cc25ec6d4c06 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -22,7 +22,8 @@ #include #include #include -#include +#include +#include #include #include #include -- cgit v1.2.3 From 669a047b1812a980a4f183a0f2a70de24c187af0 Mon Sep 17 00:00:00 2001 From: Paul Gortmaker Date: Sat, 1 Dec 2018 14:19:10 -0500 Subject: iommu/rockchip: Make it explicitly non-modular The Kconfig currently controlling compilation of this code is: drivers/iommu/Kconfig:config ROCKCHIP_IOMMU drivers/iommu/Kconfig: bool "Rockchip IOMMU Support" ...meaning that it currently is not being built as a module by anyone. The bind/unbind/remove was already explicitly disabled in commit 98b72b94def9 ("iommu/rockchip: Prohibit unbind and remove"). Lets remove the remaining traces of modular infrastructure, so that when reading the driver there is no doubt it is builtin-only. Since module_init was not in use by this code, the init ordering remains unchanged with this commit. Also note that MODULE_DEVICE_TABLE is a no-op for non-modular code. We also delete the MODULE_LICENSE tag etc. since all that information was (or is now) contained at the top of the file in the comments. Cc: Joerg Roedel Cc: Heiko Stuebner Cc: Simon Xue Cc: Daniel Kurtz Cc: Jeffy Chen Cc: iommu@lists.linux-foundation.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-rockchip@lists.infradead.org Acked-by: Heiko Stuebner Signed-off-by: Paul Gortmaker Signed-off-by: Joerg Roedel --- drivers/iommu/rockchip-iommu.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index ad3e2b97469e..c9ba9f377f63 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -1,4 +1,9 @@ /* + * IOMMU API for Rockchip + * + * Module Authors: Simon Xue + * Daniel Kurtz + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. @@ -17,7 +22,7 @@ #include #include #include -#include +#include #include #include #include @@ -1281,7 +1286,6 @@ static const struct of_device_id rk_iommu_dt_ids[] = { { .compatible = "rockchip,iommu" }, { /* sentinel */ } }; -MODULE_DEVICE_TABLE(of, rk_iommu_dt_ids); static struct platform_driver rk_iommu_driver = { .probe = rk_iommu_probe, @@ -1299,8 +1303,3 @@ static int __init rk_iommu_init(void) return platform_driver_register(&rk_iommu_driver); } subsys_initcall(rk_iommu_init); - -MODULE_DESCRIPTION("IOMMU API for Rockchip"); -MODULE_AUTHOR("Simon Xue and Daniel Kurtz "); -MODULE_ALIAS("platform:rockchip-iommu"); -MODULE_LICENSE("GPL v2"); -- cgit v1.2.3 From a007dd51e5598ab318d8d226a439f41f536ffbc2 Mon Sep 17 00:00:00 2001 From: Paul Gortmaker Date: Sat, 1 Dec 2018 14:19:11 -0500 Subject: iommu/msm: Make it explicitly non-modular The Kconfig currently controlling compilation of this code is: drivers/iommu/Kconfig:config MSM_IOMMU drivers/iommu/Kconfig: bool "MSM IOMMU Support" ...meaning that it currently is not being built as a module by anyone. Lets remove the modular code that is essentially orphaned, so that when reading the driver there is no doubt it is builtin-only. Since module_init was not even used by this driver, the init ordering remains unchanged with this commit. We also delete the MODULE_LICENSE tag etc. since all that information was (or is now) contained at the top of the file in the comments. Cc: Joerg Roedel Cc: Stepan Moskovchenko Cc: iommu@lists.linux-foundation.org Signed-off-by: Paul Gortmaker Signed-off-by: Joerg Roedel --- drivers/iommu/msm_iommu.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index fc5f0b53adaf..fc4270733f11 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -1,4 +1,6 @@ /* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. + * + * Author: Stepan Moskovchenko * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -17,7 +19,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include -#include +#include #include #include #include @@ -861,14 +863,5 @@ static int __init msm_iommu_driver_init(void) return ret; } - -static void __exit msm_iommu_driver_exit(void) -{ - platform_driver_unregister(&msm_iommu_driver); -} - subsys_initcall(msm_iommu_driver_init); -module_exit(msm_iommu_driver_exit); -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Stepan Moskovchenko "); -- cgit v1.2.3 From d4cf5bbd56f5b7210fae65cb60645c71b8c39425 Mon Sep 17 00:00:00 2001 From: Paul Gortmaker Date: Sat, 1 Dec 2018 14:19:12 -0500 Subject: iommu/mediatek: Make it explicitly non-modular The Kconfig currently controlling compilation of this code is: drivers/iommu/Kconfig:config MTK_IOMMU_V1 drivers/iommu/Kconfig: bool "MTK IOMMU Version 1 (M4U gen1) Support" ...meaning that it currently is not being built as a module by anyone. Lets remove the modular code that is essentially orphaned, so that when reading the driver there is no doubt it is builtin-only. Since module_init was not even used by this driver, the init ordering remains unchanged with this commit. We also delete the MODULE_LICENSE tag etc. since all that information was (or is now) contained at the top of the file in the comments. Cc: Joerg Roedel Cc: Matthias Brugger Cc: Honghui Zhang Cc: iommu@lists.linux-foundation.org Cc: linux-mediatek@lists.infradead.org Acked-by: Honghui Zhang Signed-off-by: Paul Gortmaker Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu_v1.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 0e780848f59b..6d4551e8ef85 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -1,4 +1,6 @@ /* + * IOMMU API for MTK architected m4u v1 implementations + * * Copyright (c) 2015-2016 MediaTek Inc. * Author: Honghui Zhang * @@ -35,7 +37,7 @@ #include #include #include -#include +#include #include #include #include "mtk_iommu.h" @@ -704,15 +706,4 @@ static int __init m4u_init(void) { return platform_driver_register(&mtk_iommu_driver); } - -static void __exit m4u_exit(void) -{ - return platform_driver_unregister(&mtk_iommu_driver); -} - subsys_initcall(m4u_init); -module_exit(m4u_exit); - -MODULE_DESCRIPTION("IOMMU API for MTK architected m4u v1 implementations"); -MODULE_AUTHOR("Honghui Zhang "); -MODULE_LICENSE("GPL v2"); -- cgit v1.2.3 From 8128ac3b0fe316d4bbedae54068ac24a623c08b5 Mon Sep 17 00:00:00 2001 From: Paul Gortmaker Date: Sat, 1 Dec 2018 14:19:13 -0500 Subject: iommu/ipmmu-vmsa: Make it explicitly non-modular The Kconfig currently controlling compilation of this code is: drivers/iommu/Kconfig:config IPMMU_VMSA drivers/iommu/Kconfig: bool "Renesas VMSA-compatible IPMMU" ...meaning that it currently is not being built as a module by anyone. Lets remove the modular code that is essentially orphaned, so that when reading the driver there is no doubt it is builtin-only. Since module_init was not even used by this driver, the init ordering remains unchanged with this commit. We also delete the MODULE_LICENSE tag etc. since all that information was (or is now) contained at the top of the file in the comments. Also note that MODULE_DEVICE_TABLE is a no-op for non-modular code. Cc: Joerg Roedel Cc: Laurent Pinchart Cc: iommu@lists.linux-foundation.org Reviewed-by: Laurent Pinchart Signed-off-by: Paul Gortmaker Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index b98a03189580..c67c3aa8537a 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* - * IPMMU VMSA + * IOMMU API for Renesas VMSA-compatible IPMMU + * Author: Laurent Pinchart * * Copyright (C) 2014 Renesas Electronics Corporation */ @@ -11,10 +12,10 @@ #include #include #include +#include #include #include #include -#include #include #include #include @@ -957,8 +958,6 @@ static const struct of_device_id ipmmu_of_ids[] = { }, }; -MODULE_DEVICE_TABLE(of, ipmmu_of_ids); - static int ipmmu_probe(struct platform_device *pdev) { struct ipmmu_vmsa_device *mmu; @@ -1129,15 +1128,4 @@ static int __init ipmmu_init(void) setup_done = true; return 0; } - -static void __exit ipmmu_exit(void) -{ - return platform_driver_unregister(&ipmmu_driver); -} - subsys_initcall(ipmmu_init); -module_exit(ipmmu_exit); - -MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU"); -MODULE_AUTHOR("Laurent Pinchart "); -MODULE_LICENSE("GPL v2"); -- cgit v1.2.3 From f295cf26d7766a1108d1dbfd183813b849c0f161 Mon Sep 17 00:00:00 2001 From: Paul Gortmaker Date: Sat, 1 Dec 2018 14:19:14 -0500 Subject: iommu/qcom: Make it explicitly non-modular The Kconfig currently controlling compilation of this code is: drivers/iommu/Kconfig:config MTK_IOMMU_V1 drivers/iommu/Kconfig: bool "MTK IOMMU Version 1 (M4U gen1) Support" ...meaning that it currently is not being built as a module by anyone. Lets remove the modular code that is essentially orphaned, so that when reading the driver there is no doubt it is builtin-only. Since module_init just becomes device_initcall for non-modules, the init ordering remains unchanged with this commit. Also note that MODULE_DEVICE_TABLE is a no-op for non-modular code. We also delete the MODULE_LICENSE tag etc. since all that information was (or is now) contained at the top of the file in the comments. Cc: Rob Clark Cc: Joerg Roedel Cc: iommu@lists.linux-foundation.org Cc: linux-arm-msm@vger.kernel.org Signed-off-by: Paul Gortmaker Signed-off-by: Joerg Roedel --- drivers/iommu/qcom_iommu.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c index ee70e9921cf1..4c8f4fc54106 100644 --- a/drivers/iommu/qcom_iommu.c +++ b/drivers/iommu/qcom_iommu.c @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include #include @@ -908,7 +908,6 @@ static const struct of_device_id qcom_iommu_of_match[] = { { .compatible = "qcom,msm-iommu-v1" }, { /* sentinel */ } }; -MODULE_DEVICE_TABLE(of, qcom_iommu_of_match); static struct platform_driver qcom_iommu_driver = { .driver = { @@ -934,15 +933,4 @@ static int __init qcom_iommu_init(void) return ret; } - -static void __exit qcom_iommu_exit(void) -{ - platform_driver_unregister(&qcom_iommu_driver); - platform_driver_unregister(&qcom_iommu_ctx_driver); -} - -module_init(qcom_iommu_init); -module_exit(qcom_iommu_exit); - -MODULE_DESCRIPTION("IOMMU API for QCOM IOMMU v1 implementations"); -MODULE_LICENSE("GPL v2"); +device_initcall(qcom_iommu_init); -- cgit v1.2.3 From 39fcbbccf8ed01e5b562136a97434d15d1fd039b Mon Sep 17 00:00:00 2001 From: Paul Gortmaker Date: Sat, 1 Dec 2018 14:19:15 -0500 Subject: iommu/tegra: Make it explicitly non-modular The Kconfig currently controlling compilation of this code is: drivers/iommu/Kconfig:config TEGRA_IOMMU_GART drivers/iommu/Kconfig: bool "Tegra GART IOMMU Support" ...meaning that it currently is not being built as a module by anyone. Lets remove the modular code that is essentially orphaned, so that when reading the driver there is no doubt it is builtin-only. We explicitly disallow a driver unbind, since that doesn't have a sensible use case anyway, and it allows us to drop the ".remove" code for non-modular drivers. Since module_init was not in use by this code, the init ordering remains unchanged with this commit. We replace module.h with moduleparam.h since the file does actually declare some module_param() and the easiest way to keep back compatibility with existing use cases is to leave it as-is for now. The init function was missing an __init annotation, so it was added. We also delete the MODULE_LICENSE tag etc. since all that information was (or is now) contained at the top of the file in the comments. Cc: Hiroshi Doyu Cc: Joerg Roedel Cc: Stephen Warren Cc: Thierry Reding Cc: Alexandre Courbot Cc: iommu@lists.linux-foundation.org Cc: linux-tegra@vger.kernel.org Acked-by: Thierry Reding Signed-off-by: Paul Gortmaker Signed-off-by: Joerg Roedel --- drivers/iommu/tegra-gart.c | 37 +++++++------------------------------ 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c index 7b1361d57a17..da6a4e357b2b 100644 --- a/drivers/iommu/tegra-gart.c +++ b/drivers/iommu/tegra-gart.c @@ -3,6 +3,8 @@ * * Copyright (c) 2010-2012, NVIDIA CORPORATION. All rights reserved. * + * Author: Hiroshi DOYU + * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. @@ -19,7 +21,8 @@ #define pr_fmt(fmt) "%s(): " fmt, __func__ -#include +#include +#include #include #include #include @@ -478,20 +481,6 @@ static int tegra_gart_probe(struct platform_device *pdev) return 0; } -static int tegra_gart_remove(struct platform_device *pdev) -{ - struct gart_device *gart = platform_get_drvdata(pdev); - - iommu_device_unregister(&gart->iommu); - iommu_device_sysfs_remove(&gart->iommu); - - writel(0, gart->regs + GART_CONFIG); - if (gart->savedata) - vfree(gart->savedata); - gart_handle = NULL; - return 0; -} - static const struct dev_pm_ops tegra_gart_pm_ops = { .suspend = tegra_gart_suspend, .resume = tegra_gart_resume, @@ -501,34 +490,22 @@ static const struct of_device_id tegra_gart_of_match[] = { { .compatible = "nvidia,tegra20-gart", }, { }, }; -MODULE_DEVICE_TABLE(of, tegra_gart_of_match); static struct platform_driver tegra_gart_driver = { .probe = tegra_gart_probe, - .remove = tegra_gart_remove, .driver = { .name = "tegra-gart", .pm = &tegra_gart_pm_ops, .of_match_table = tegra_gart_of_match, + .suppress_bind_attrs = true, }, }; -static int tegra_gart_init(void) +static int __init tegra_gart_init(void) { return platform_driver_register(&tegra_gart_driver); } - -static void __exit tegra_gart_exit(void) -{ - platform_driver_unregister(&tegra_gart_driver); -} - subsys_initcall(tegra_gart_init); -module_exit(tegra_gart_exit); -module_param(gart_debug, bool, 0644); +module_param(gart_debug, bool, 0644); MODULE_PARM_DESC(gart_debug, "Enable GART debugging"); -MODULE_DESCRIPTION("IOMMU API for GART in Tegra20"); -MODULE_AUTHOR("Hiroshi DOYU "); -MODULE_ALIAS("platform:tegra-gart"); -MODULE_LICENSE("GPL v2"); -- cgit v1.2.3 From addb672f200f4e99368270da205320b83efe01a0 Mon Sep 17 00:00:00 2001 From: Paul Gortmaker Date: Sat, 1 Dec 2018 14:19:16 -0500 Subject: iommu/arm-smmu: Make arm-smmu explicitly non-modular The Kconfig currently controlling compilation of this code is: drivers/iommu/Kconfig:config ARM_SMMU drivers/iommu/Kconfig: bool "ARM Ltd. System MMU (SMMU) Support" ...meaning that it currently is not being built as a module by anyone. Lets remove the modular code that is essentially orphaned, so that when reading the driver there is no doubt it is builtin-only. Since module_platform_driver() uses the same init level priority as builtin_platform_driver() the init ordering remains unchanged with this commit. We explicitly disallow a driver unbind, since that doesn't have a sensible use case anyway, but unlike most drivers, we can't delete the function tied to the ".remove" field. This is because as of commit 7aa8619a66ae ("iommu/arm-smmu-v3: Implement shutdown method") the .remove function was given a one line wrapper and re-used to provide a .shutdown service. So we delete the wrapper and re-name the function from remove to shutdown. We add a moduleparam.h include since the file does actually declare some module parameters, and leaving them as such is the easiest way currently to remain backwards compatible with existing use cases. We also delete the MODULE_LICENSE tag etc. since all that information is already contained at the top of the file in the comments. Cc: Will Deacon Cc: Joerg Roedel Cc: Robin Murphy Cc: Nate Watterson Cc: linux-arm-kernel@lists.infradead.org Cc: iommu@lists.linux-foundation.org Acked-by: Robin Murphy Signed-off-by: Paul Gortmaker Signed-off-by: Joerg Roedel --- drivers/iommu/arm-smmu.c | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 5a28ae892504..4a2e143fdf52 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -41,7 +41,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -101,6 +102,10 @@ #define MSI_IOVA_LENGTH 0x100000 static int force_stage; +/* + * not really modular, but the easiest way to keep compat with existing + * bootargs behaviour is to continue using module_param() here. + */ module_param(force_stage, int, S_IRUGO); MODULE_PARM_DESC(force_stage, "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); @@ -1964,7 +1969,6 @@ static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, { }, }; -MODULE_DEVICE_TABLE(of, arm_smmu_of_match); #ifdef CONFIG_ACPI static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) @@ -2224,24 +2228,18 @@ static int arm_smmu_legacy_bus_init(void) } device_initcall_sync(arm_smmu_legacy_bus_init); -static int arm_smmu_device_remove(struct platform_device *pdev) +static void arm_smmu_device_shutdown(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); if (!smmu) - return -ENODEV; + return; if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) dev_err(&pdev->dev, "removing device with active domains!\n"); /* Turn the thing off */ writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); - return 0; -} - -static void arm_smmu_device_shutdown(struct platform_device *pdev) -{ - arm_smmu_device_remove(pdev); } static int __maybe_unused arm_smmu_pm_resume(struct device *dev) @@ -2256,16 +2254,12 @@ static SIMPLE_DEV_PM_OPS(arm_smmu_pm_ops, NULL, arm_smmu_pm_resume); static struct platform_driver arm_smmu_driver = { .driver = { - .name = "arm-smmu", - .of_match_table = of_match_ptr(arm_smmu_of_match), - .pm = &arm_smmu_pm_ops, + .name = "arm-smmu", + .of_match_table = of_match_ptr(arm_smmu_of_match), + .pm = &arm_smmu_pm_ops, + .suppress_bind_attrs = true, }, .probe = arm_smmu_device_probe, - .remove = arm_smmu_device_remove, .shutdown = arm_smmu_device_shutdown, }; -module_platform_driver(arm_smmu_driver); - -MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); -MODULE_AUTHOR("Will Deacon "); -MODULE_LICENSE("GPL v2"); +builtin_platform_driver(arm_smmu_driver); -- cgit v1.2.3 From c07b6426df922d21a13a959cf785d46e9c531941 Mon Sep 17 00:00:00 2001 From: Paul Gortmaker Date: Sat, 1 Dec 2018 14:19:17 -0500 Subject: iommu/arm-smmu: Make arm-smmu-v3 explicitly non-modular The Kconfig currently controlling compilation of this code is: drivers/iommu/Kconfig:config ARM_SMMU_V3 drivers/iommu/Kconfig: bool "ARM Ltd. System MMU Version 3 (SMMUv3) Support" ...meaning that it currently is not being built as a module by anyone. Lets remove the modular code that is essentially orphaned, so that when reading the driver there is no doubt it is builtin-only. Since module_platform_driver() uses the same init level priority as builtin_platform_driver() the init ordering remains unchanged with this commit. We explicitly disallow a driver unbind, since that doesn't have a sensible use case anyway, but unlike most drivers, we can't delete the function tied to the ".remove" field. This is because as of commit 7aa8619a66ae ("iommu/arm-smmu-v3: Implement shutdown method") the .remove function was given a one line wrapper and re-used to provide a .shutdown service. So we delete the wrapper and re-name the function from remove to shutdown. We add a moduleparam.h include since the file does actually declare some module parameters, and leaving them as such is the easiest way currently to remain backwards compatible with existing use cases. Also note that MODULE_DEVICE_TABLE is a no-op for non-modular code. We also delete the MODULE_LICENSE tag etc. since all that information is already contained at the top of the file in the comments. Cc: Will Deacon Cc: Joerg Roedel Cc: Robin Murphy Cc: Nate Watterson Cc: linux-arm-kernel@lists.infradead.org Cc: iommu@lists.linux-foundation.org Acked-by: Robin Murphy Signed-off-by: Paul Gortmaker Signed-off-by: Joerg Roedel --- drivers/iommu/arm-smmu-v3.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 6947ccf26512..1189c06079d4 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -20,7 +20,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -356,6 +357,10 @@ #define MSI_IOVA_BASE 0x8000000 #define MSI_IOVA_LENGTH 0x100000 +/* + * not really modular, but the easiest way to keep compat with existing + * bootargs behaviour is to continue using module_param_named here. + */ static bool disable_bypass = 1; module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO); MODULE_PARM_DESC(disable_bypass, @@ -2928,37 +2933,25 @@ static int arm_smmu_device_probe(struct platform_device *pdev) return 0; } -static int arm_smmu_device_remove(struct platform_device *pdev) +static void arm_smmu_device_shutdown(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); arm_smmu_device_disable(smmu); - - return 0; -} - -static void arm_smmu_device_shutdown(struct platform_device *pdev) -{ - arm_smmu_device_remove(pdev); } static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,smmu-v3", }, { }, }; -MODULE_DEVICE_TABLE(of, arm_smmu_of_match); static struct platform_driver arm_smmu_driver = { .driver = { .name = "arm-smmu-v3", .of_match_table = of_match_ptr(arm_smmu_of_match), + .suppress_bind_attrs = true, }, .probe = arm_smmu_device_probe, - .remove = arm_smmu_device_remove, .shutdown = arm_smmu_device_shutdown, }; -module_platform_driver(arm_smmu_driver); - -MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations"); -MODULE_AUTHOR("Will Deacon "); -MODULE_LICENSE("GPL v2"); +builtin_platform_driver(arm_smmu_driver); -- cgit v1.2.3 From 51eb78098ab79bba8b1df24da2304e61deb74629 Mon Sep 17 00:00:00 2001 From: tom Date: Tue, 4 Dec 2018 18:27:34 +0000 Subject: iommu: Change tlb_range_add to iotlb_range_add and tlb_sync to iotlb_sync Someone forgot to update this comment. Signed-off-by: Tom Murphy Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index a1d28f42cb77..11db18b9ffe8 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -168,8 +168,8 @@ struct iommu_resv_region { * @map: map a physically contiguous memory region to an iommu domain * @unmap: unmap a physically contiguous memory region from an iommu domain * @flush_tlb_all: Synchronously flush all hardware TLBs for this domain - * @tlb_range_add: Add a given iova range to the flush queue for this domain - * @tlb_sync: Flush all queued ranges from the hardware TLBs and empty flush + * @iotlb_range_add: Add a given iova range to the flush queue for this domain + * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush * queue * @iova_to_phys: translate iova to physical address * @add_device: add device to iommu grouping -- cgit v1.2.3 From c12b08ebbe16f0d3a96a116d86709b04c1ee8e74 Mon Sep 17 00:00:00 2001 From: Yu Zhao Date: Thu, 6 Dec 2018 14:39:15 -0700 Subject: iommu/amd: Fix amd_iommu=force_isolation The parameter is still there but it's ignored. We need to check its value before deciding to go into passthrough mode for AMD IOMMU v2 capable device. We occasionally use this parameter to force v2 capable device into translation mode to debug memory corruption that we suspect is caused by DMA writes. To address the following comment from Joerg Roedel on the first version, v2 capability of device is completely ignored. > This breaks the iommu_v2 use-case, as it needs a direct mapping for the > devices that support it. And from Documentation/admin-guide/kernel-parameters.txt: This option does not override iommu=pt Fixes: aafd8ba0ca74 ("iommu/amd: Implement add_device and remove_device") Signed-off-by: Yu Zhao Signed-off-by: Joerg Roedel --- drivers/iommu/amd_iommu.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index c5d78c87248a..49a669b1ce11 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -440,7 +440,14 @@ static int iommu_init_device(struct device *dev) dev_data->alias = get_alias(dev); - if (dev_is_pci(dev) && pci_iommuv2_capable(to_pci_dev(dev))) { + /* + * By default we use passthrough mode for IOMMUv2 capable device. + * But if amd_iommu=force_isolation is set (e.g. to debug DMA to + * invalid address), we ignore the capability for the device so + * it'll be forced to go into translation mode. + */ + if ((iommu_pass_through || !amd_iommu_force_isolation) && + dev_is_pci(dev) && pci_iommuv2_capable(to_pci_dev(dev))) { struct amd_iommu *iommu; iommu = amd_iommu_rlookup_table[dev_data->devid]; -- cgit v1.2.3 From 3cd508a8c1379427afb5e16c2e0a7c986d907853 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 17 Oct 2018 21:32:58 +0100 Subject: iommu/arm-smmu-v3: Fix big-endian CMD_SYNC writes When we insert the sync sequence number into the CMD_SYNC.MSIData field, we do so in CPU-native byte order, before writing out the whole command as explicitly little-endian dwords. Thus on big-endian systems, the SMMU will receive and write back a byteswapped version of sync_nr, which would be perfect if it were targeting a similarly-little-endian ITS, but since it's actually writing back to memory being polled by the CPUs, they're going to end up seeing the wrong thing. Since the SMMU doesn't care what the MSIData actually contains, the minimal-overhead solution is to simply add an extra byteswap initially, such that it then writes back the big-endian format directly. Cc: Fixes: 37de98f8f1cf ("iommu/arm-smmu-v3: Use CMD_SYNC completion MSI") Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 6947ccf26512..71eda422c926 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -828,7 +828,13 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV); cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH); cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB); - cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIDATA, ent->sync.msidata); + /* + * Commands are written little-endian, but we want the SMMU to + * receive MSIData, and thus write it back to memory, in CPU + * byte order, so big-endian needs an extra byteswap here. + */ + cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIDATA, + cpu_to_le32(ent->sync.msidata)); cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK; break; default: -- cgit v1.2.3 From 84a9a75774961612d0c7dd34a1777e8f98a65abd Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Wed, 31 Oct 2018 12:02:07 +0800 Subject: iommu/arm-smmu-v3: Avoid memory corruption from Hisilicon MSI payloads The GITS_TRANSLATER MMIO doorbell register in the ITS hardware is architected to be 4 bytes in size, yet on hi1620 and earlier, Hisilicon have allocated the adjacent 4 bytes to carry some IMPDEF sideband information which results in an 8-byte MSI payload being delivered when signalling an interrupt: MSIAddr: |----4bytes----|----4bytes----| | MSIData | IMPDEF | This poses no problem for the ITS hardware because the adjacent 4 bytes are reserved in the memory map. However, when delivering MSIs to memory, as we do in the SMMUv3 driver for signalling the completion of a SYNC command, the extended payload will corrupt the 4 bytes adjacent to the "sync_count" member in struct arm_smmu_device. Fortunately, the current layout allocates these bytes to padding, but this is fragile and we should make this explicit. Reviewed-by: Robin Murphy Signed-off-by: Zhen Lei [will: Rewrote commit message and comment] Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 71eda422c926..62ef4afc9ee5 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -576,7 +576,11 @@ struct arm_smmu_device { struct arm_smmu_strtab_cfg strtab_cfg; - u32 sync_count; + /* Hi16xx adds an extra 32 bits of goodness to its MSI payload */ + union { + u32 sync_count; + u64 padding; + }; /* IOMMU core code handle */ struct iommu_device iommu; -- cgit v1.2.3 From a868e8530441286342f90c1fd9c5f24de3aa2880 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 7 Nov 2018 22:58:24 +0000 Subject: iommu/arm-smmu-v3: Use explicit mb() when moving cons pointer After removing an entry from a queue (e.g. reading an event in arm_smmu_evtq_thread()) it is necessary to advance the MMIO consumer pointer to free the queue slot back to the SMMU. A memory barrier is required here so that all reads targetting the queue entry have completed before the consumer pointer is updated. The implementation of queue_inc_cons() relies on a writel() to complete the previous reads, but this is incorrect because writel() is only guaranteed to complete prior writes. This patch replaces the call to writel() with an mb(); writel_relaxed() sequence, which gives us the read->write ordering which we require. Cc: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu-v3.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 62ef4afc9ee5..11f528e727a1 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -679,7 +679,13 @@ static void queue_inc_cons(struct arm_smmu_queue *q) u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1; q->cons = Q_OVF(q, q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons); - writel(q->cons, q->cons_reg); + + /* + * Ensure that all CPU accesses (reads and writes) to the queue + * are complete before we update the cons pointer. + */ + mb(); + writel_relaxed(q->cons, q->cons_reg); } static int queue_sync_prod(struct arm_smmu_queue *q) -- cgit v1.2.3 From 96a299d24cfb6b9659304131b598fa09f4c9041e Mon Sep 17 00:00:00 2001 From: Sricharan R Date: Tue, 4 Dec 2018 11:52:09 +0530 Subject: iommu/arm-smmu: Add pm_runtime/sleep ops The smmu needs to be functional only when the respective master's using it are active. The device_link feature helps to track such functional dependencies, so that the iommu gets powered when the master device enables itself using pm_runtime. So by adapting the smmu driver for runtime pm, above said dependency can be addressed. This patch adds the pm runtime/sleep callbacks to the driver and the corresponding bulk clock handling for all the clocks needed by smmu. Also, while we enable the runtime pm, add a pm sleep suspend callback that pushes devices to low power state by turning the clocks off in a system sleep. Add corresponding clock enable path in resume callback as well. Signed-off-by: Sricharan R Signed-off-by: Archit Taneja [Thor: Rework to get clocks from device tree] Signed-off-by: Thor Thayer [vivek: rework for clock and pm ops] Signed-off-by: Vivek Gautam Reviewed-by: Tomasz Figa Tested-by: Srinivas Kandagatla Reviewed-by: Robin Murphy Tested-by: Thor Thayer Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 58 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 5a28ae892504..602b67d4f2d6 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include @@ -206,6 +207,8 @@ struct arm_smmu_device { u32 num_global_irqs; u32 num_context_irqs; unsigned int *irqs; + struct clk_bulk_data *clks; + int num_clks; u32 cavium_id_base; /* Specific to Cavium */ @@ -1947,7 +1950,7 @@ struct arm_smmu_match_data { }; #define ARM_SMMU_MATCH_DATA(name, ver, imp) \ -static struct arm_smmu_match_data name = { .version = ver, .model = imp } +static const struct arm_smmu_match_data name = { .version = ver, .model = imp } ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); @@ -2150,6 +2153,17 @@ static int arm_smmu_device_probe(struct platform_device *pdev) smmu->irqs[i] = irq; } + err = devm_clk_bulk_get_all(dev, &smmu->clks); + if (err < 0) { + dev_err(dev, "failed to get clocks %d\n", err); + return err; + } + smmu->num_clks = err; + + err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks); + if (err) + return err; + err = arm_smmu_device_cfg_probe(smmu); if (err) return err; @@ -2236,6 +2250,9 @@ static int arm_smmu_device_remove(struct platform_device *pdev) /* Turn the thing off */ writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); + + clk_bulk_disable_unprepare(smmu->num_clks, smmu->clks); + return 0; } @@ -2244,15 +2261,50 @@ static void arm_smmu_device_shutdown(struct platform_device *pdev) arm_smmu_device_remove(pdev); } -static int __maybe_unused arm_smmu_pm_resume(struct device *dev) +static int __maybe_unused arm_smmu_runtime_resume(struct device *dev) { struct arm_smmu_device *smmu = dev_get_drvdata(dev); + int ret; + + ret = clk_bulk_enable(smmu->num_clks, smmu->clks); + if (ret) + return ret; arm_smmu_device_reset(smmu); + return 0; } -static SIMPLE_DEV_PM_OPS(arm_smmu_pm_ops, NULL, arm_smmu_pm_resume); +static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev) +{ + struct arm_smmu_device *smmu = dev_get_drvdata(dev); + + clk_bulk_disable(smmu->num_clks, smmu->clks); + + return 0; +} + +static int __maybe_unused arm_smmu_pm_resume(struct device *dev) +{ + if (pm_runtime_suspended(dev)) + return 0; + + return arm_smmu_runtime_resume(dev); +} + +static int __maybe_unused arm_smmu_pm_suspend(struct device *dev) +{ + if (pm_runtime_suspended(dev)) + return 0; + + return arm_smmu_runtime_suspend(dev); +} + +static const struct dev_pm_ops arm_smmu_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume) + SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend, + arm_smmu_runtime_resume, NULL) +}; static struct platform_driver arm_smmu_driver = { .driver = { -- cgit v1.2.3 From d4a44f0750bb9bf7c44c22444a482756b2344a79 Mon Sep 17 00:00:00 2001 From: Sricharan R Date: Tue, 4 Dec 2018 11:52:10 +0530 Subject: iommu/arm-smmu: Invoke pm_runtime across the driver Enable pm-runtime on devices that implement a pm domain. Then, add pm runtime hooks to several iommu_ops to power cycle the smmu device for explicit TLB invalidation requests, and register space accesses, etc. We need these hooks when the smmu, linked to its master through device links, has to be powered-up without the master device being in context. Signed-off-by: Sricharan R [vivek: Cleanup pm runtime calls] Signed-off-by: Vivek Gautam Reviewed-by: Tomasz Figa Tested-by: Srinivas Kandagatla Reviewed-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 108 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 98 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 602b67d4f2d6..1917d214c4d9 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -270,6 +270,20 @@ static struct arm_smmu_option_prop arm_smmu_options[] = { { 0, NULL}, }; +static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu) +{ + if (pm_runtime_enabled(smmu->dev)) + return pm_runtime_get_sync(smmu->dev); + + return 0; +} + +static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu) +{ + if (pm_runtime_enabled(smmu->dev)) + pm_runtime_put(smmu->dev); +} + static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) { return container_of(dom, struct arm_smmu_domain, domain); @@ -929,11 +943,15 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - int irq; + int ret, irq; if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY) return; + ret = arm_smmu_rpm_get(smmu); + if (ret < 0) + return; + /* * Disable the context bank and free the page tables before freeing * it. @@ -948,6 +966,8 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) free_io_pgtable_ops(smmu_domain->pgtbl_ops); __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); + + arm_smmu_rpm_put(smmu); } static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) @@ -1229,10 +1249,15 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) return -ENODEV; smmu = fwspec_smmu(fwspec); + + ret = arm_smmu_rpm_get(smmu); + if (ret < 0) + return ret; + /* Ensure that the domain is finalised */ ret = arm_smmu_init_domain_context(domain, smmu); if (ret < 0) - return ret; + goto rpm_put; /* * Sanity check the domain. We don't support domains across @@ -1242,49 +1267,74 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) dev_err(dev, "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n", dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev)); - return -EINVAL; + ret = -EINVAL; + goto rpm_put; } /* Looks ok, so add the device to the domain */ - return arm_smmu_domain_add_master(smmu_domain, fwspec); + ret = arm_smmu_domain_add_master(smmu_domain, fwspec); + +rpm_put: + arm_smmu_rpm_put(smmu); + return ret; } static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; + struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; + int ret; if (!ops) return -ENODEV; - return ops->map(ops, iova, paddr, size, prot); + arm_smmu_rpm_get(smmu); + ret = ops->map(ops, iova, paddr, size, prot); + arm_smmu_rpm_put(smmu); + + return ret; } static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) { struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; + struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; + size_t ret; if (!ops) return 0; - return ops->unmap(ops, iova, size); + arm_smmu_rpm_get(smmu); + ret = ops->unmap(ops, iova, size); + arm_smmu_rpm_put(smmu); + + return ret; } static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; - if (smmu_domain->tlb_ops) + if (smmu_domain->tlb_ops) { + arm_smmu_rpm_get(smmu); smmu_domain->tlb_ops->tlb_flush_all(smmu_domain); + arm_smmu_rpm_put(smmu); + } } static void arm_smmu_iotlb_sync(struct iommu_domain *domain) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; - if (smmu_domain->tlb_ops) + if (smmu_domain->tlb_ops) { + arm_smmu_rpm_get(smmu); smmu_domain->tlb_ops->tlb_sync(smmu_domain); + arm_smmu_rpm_put(smmu); + } } static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, @@ -1299,6 +1349,11 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, u32 tmp; u64 phys; unsigned long va, flags; + int ret; + + ret = arm_smmu_rpm_get(smmu); + if (ret < 0) + return 0; cb_base = ARM_SMMU_CB(smmu, cfg->cbndx); @@ -1327,6 +1382,8 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, return 0; } + arm_smmu_rpm_put(smmu); + return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); } @@ -1431,7 +1488,13 @@ static int arm_smmu_add_device(struct device *dev) while (i--) cfg->smendx[i] = INVALID_SMENDX; + ret = arm_smmu_rpm_get(smmu); + if (ret < 0) + goto out_cfg_free; + ret = arm_smmu_master_alloc_smes(dev); + arm_smmu_rpm_put(smmu); + if (ret) goto out_cfg_free; @@ -1451,7 +1514,7 @@ static void arm_smmu_remove_device(struct device *dev) struct iommu_fwspec *fwspec = dev->iommu_fwspec; struct arm_smmu_master_cfg *cfg; struct arm_smmu_device *smmu; - + int ret; if (!fwspec || fwspec->ops != &arm_smmu_ops) return; @@ -1459,8 +1522,15 @@ static void arm_smmu_remove_device(struct device *dev) cfg = fwspec->iommu_priv; smmu = cfg->smmu; + ret = arm_smmu_rpm_get(smmu); + if (ret < 0) + return; + iommu_device_unlink(&smmu->iommu, dev); arm_smmu_master_free_smes(fwspec); + + arm_smmu_rpm_put(smmu); + iommu_group_remove_device(dev); kfree(fwspec->iommu_priv); iommu_fwspec_free(dev); @@ -2213,6 +2283,17 @@ static int arm_smmu_device_probe(struct platform_device *pdev) arm_smmu_device_reset(smmu); arm_smmu_test_smr_masks(smmu); + /* + * We want to avoid touching dev->power.lock in fastpaths unless + * it's really going to do something useful - pm_runtime_enabled() + * can serve as an ideal proxy for that decision. So, conditionally + * enable pm_runtime. + */ + if (dev->pm_domain) { + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + } + /* * For ACPI and generic DT bindings, an SMMU will be probed before * any device which might need it, so we want the bus ops in place @@ -2248,10 +2329,17 @@ static int arm_smmu_device_remove(struct platform_device *pdev) if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) dev_err(&pdev->dev, "removing device with active domains!\n"); + arm_smmu_rpm_get(smmu); /* Turn the thing off */ writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); + arm_smmu_rpm_put(smmu); + + if (pm_runtime_enabled(smmu->dev)) + pm_runtime_force_suspend(smmu->dev); + else + clk_bulk_disable(smmu->num_clks, smmu->clks); - clk_bulk_disable_unprepare(smmu->num_clks, smmu->clks); + clk_bulk_unprepare(smmu->num_clks, smmu->clks); return 0; } -- cgit v1.2.3 From 655e364374e74a910d87dbe758bad3e4794f659e Mon Sep 17 00:00:00 2001 From: Sricharan R Date: Tue, 4 Dec 2018 11:52:11 +0530 Subject: iommu/arm-smmu: Add the device_link between masters and smmu Finally add the device link between the master device and smmu, so that the smmu gets runtime enabled/disabled only when the master needs it. This is done from add_device callback which gets called once when the master is added to the smmu. Signed-off-by: Sricharan R Signed-off-by: Vivek Gautam Reviewed-by: Tomasz Figa Tested-by: Srinivas Kandagatla Reviewed-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 1917d214c4d9..b6b11642b3a9 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1500,6 +1500,9 @@ static int arm_smmu_add_device(struct device *dev) iommu_device_link(&smmu->iommu, dev); + device_link_add(dev, smmu->dev, + DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); + return 0; out_cfg_free: -- cgit v1.2.3 From f1518cd84b44d6c28fb40e0551a9785bc38f55af Mon Sep 17 00:00:00 2001 From: Vivek Gautam Date: Tue, 4 Dec 2018 11:52:12 +0530 Subject: dt-bindings: arm-smmu: Add bindings for qcom,smmu-v2 Add bindings doc for Qcom's smmu-v2 implementation. Signed-off-by: Vivek Gautam Reviewed-by: Tomasz Figa Tested-by: Srinivas Kandagatla Reviewed-by: Rob Herring Reviewed-by: Robin Murphy Signed-off-by: Will Deacon --- .../devicetree/bindings/iommu/arm,smmu.txt | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.txt b/Documentation/devicetree/bindings/iommu/arm,smmu.txt index 8a6ffce12af5..a6504b37cc21 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.txt +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.txt @@ -17,10 +17,16 @@ conditions. "arm,mmu-401" "arm,mmu-500" "cavium,smmu-v2" + "qcom,smmu-v2" depending on the particular implementation and/or the version of the architecture implemented. + Qcom SoCs must contain, as below, SoC-specific compatibles + along with "qcom,smmu-v2": + "qcom,msm8996-smmu-v2", "qcom,smmu-v2", + "qcom,sdm845-smmu-v2", "qcom,smmu-v2". + - reg : Base address and size of the SMMU. - #global-interrupts : The number of global interrupts exposed by the @@ -71,6 +77,22 @@ conditions. or using stream matching with #iommu-cells = <2>, and may be ignored if present in such cases. +- clock-names: List of the names of clocks input to the device. The + required list depends on particular implementation and + is as follows: + - for "qcom,smmu-v2": + - "bus": clock required for downstream bus access and + for the smmu ptw, + - "iface": clock required to access smmu's registers + through the TCU's programming interface. + - unspecified for other implementations. + +- clocks: Specifiers for all clocks listed in the clock-names property, + as per generic clock bindings. + +- power-domains: Specifiers for power domains required to be powered on for + the SMMU to operate, as per generic power domain bindings. + ** Deprecated properties: - mmu-masters (deprecated in favour of the generic "iommus" binding) : @@ -137,3 +159,20 @@ conditions. iommu-map = <0 &smmu3 0 0x400>; ... }; + + /* Qcom's arm,smmu-v2 implementation */ + smmu4: iommu@d00000 { + compatible = "qcom,msm8996-smmu-v2", "qcom,smmu-v2"; + reg = <0xd00000 0x10000>; + + #global-interrupts = <1>; + interrupts = , + , + ; + #iommu-cells = <1>; + power-domains = <&mmcc MDSS_GDSC>; + + clocks = <&mmcc SMMU_MDP_AXI_CLK>, + <&mmcc SMMU_MDP_AHB_CLK>; + clock-names = "bus", "iface"; + }; -- cgit v1.2.3 From 89cddc563743cb1e0068867ac97013b2a5bf86aa Mon Sep 17 00:00:00 2001 From: Vivek Gautam Date: Tue, 4 Dec 2018 11:52:13 +0530 Subject: iommu/arm-smmu: Add support for qcom,smmu-v2 variant qcom,smmu-v2 is an arm,smmu-v2 implementation with specific clock and power requirements. On msm8996, multiple cores, viz. mdss, video, etc. use this smmu. On sdm845, this smmu is used with gpu. Add bindings for the same. Signed-off-by: Vivek Gautam Reviewed-by: Rob Herring Reviewed-by: Tomasz Figa Tested-by: Srinivas Kandagatla Reviewed-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/iommu/arm-smmu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index b6b11642b3a9..ba18d89d4732 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -120,6 +120,7 @@ enum arm_smmu_implementation { GENERIC_SMMU, ARM_MMU500, CAVIUM_SMMUV2, + QCOM_SMMUV2, }; struct arm_smmu_s2cr { @@ -2030,6 +2031,7 @@ ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); +ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2); static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, @@ -2038,6 +2040,7 @@ static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,mmu-401", .data = &arm_mmu401 }, { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, + { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 }, { }, }; MODULE_DEVICE_TABLE(of, arm_smmu_of_match); -- cgit v1.2.3 From c4b17afb0a4e8d042320efaf2acf55cb26795f78 Mon Sep 17 00:00:00 2001 From: Ganapatrao Kulkarni Date: Fri, 30 Nov 2018 19:14:00 +0800 Subject: iommu/dma: Use NUMA aware memory allocations in __iommu_dma_alloc_pages() Change function __iommu_dma_alloc_pages() to allocate pages for DMA from respective device NUMA node. The ternary operator which would be for alloc_pages_node() is tidied along with this. The motivation for this change is to have a policy for page allocation consistent with direct DMA mapping, which attempts to allocate pages local to the device, as mentioned in [1]. In addition, for certain workloads it has been observed a marginal performance improvement. The patch caused an observation of 0.9% average throughput improvement for running tcrypt with HiSilicon crypto engine. We also include a modification to use kvzalloc() for kzalloc()/vzalloc() combination. [1] https://www.mail-archive.com/linux-kernel@vger.kernel.org/msg1692998.html Signed-off-by: Ganapatrao Kulkarni [JPG: Added kvzalloc(), drop pages ** being device local, remove ternary operator, update message] Signed-off-by: John Garry Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index d1b04753b204..4afb1a8a8da8 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -449,20 +449,17 @@ static void __iommu_dma_free_pages(struct page **pages, int count) kvfree(pages); } -static struct page **__iommu_dma_alloc_pages(unsigned int count, - unsigned long order_mask, gfp_t gfp) +static struct page **__iommu_dma_alloc_pages(struct device *dev, + unsigned int count, unsigned long order_mask, gfp_t gfp) { struct page **pages; - unsigned int i = 0, array_size = count * sizeof(*pages); + unsigned int i = 0, nid = dev_to_node(dev); order_mask &= (2U << MAX_ORDER) - 1; if (!order_mask) return NULL; - if (array_size <= PAGE_SIZE) - pages = kzalloc(array_size, GFP_KERNEL); - else - pages = vzalloc(array_size); + pages = kvzalloc(count * sizeof(*pages), GFP_KERNEL); if (!pages) return NULL; @@ -481,10 +478,12 @@ static struct page **__iommu_dma_alloc_pages(unsigned int count, for (order_mask &= (2U << __fls(count)) - 1; order_mask; order_mask &= ~order_size) { unsigned int order = __fls(order_mask); + gfp_t alloc_flags = gfp; order_size = 1U << order; - page = alloc_pages((order_mask - order_size) ? - gfp | __GFP_NORETRY : gfp, order); + if (order_mask > order_size) + alloc_flags |= __GFP_NORETRY; + page = alloc_pages_node(nid, alloc_flags, order); if (!page) continue; if (!order) @@ -569,7 +568,8 @@ struct page **iommu_dma_alloc(struct device *dev, size_t size, gfp_t gfp, alloc_sizes = min_size; count = PAGE_ALIGN(size) >> PAGE_SHIFT; - pages = __iommu_dma_alloc_pages(count, alloc_sizes >> PAGE_SHIFT, gfp); + pages = __iommu_dma_alloc_pages(dev, count, alloc_sizes >> PAGE_SHIFT, + gfp); if (!pages) return NULL; -- cgit v1.2.3 From 765b6a98c1de3d84dfdae344cc4ee4c24d9447f7 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 10 Dec 2018 09:58:55 +0800 Subject: iommu/vt-d: Enumerate the scalable mode capability The Intel vt-d spec rev3.0 introduces a new translation mode called scalable mode, which enables PASID-granular translations for first level, second level, nested and pass-through modes. At the same time, the previous Extended Context (ECS) mode is deprecated (no production ever implements ECS). This patch adds enumeration for Scalable Mode and removes the deprecated ECS enumeration. It provides a boot time option to disable scalable mode even hardware claims to support it. Cc: Ashok Raj Cc: Jacob Pan Cc: Kevin Tian Signed-off-by: Sanjay Kumar Signed-off-by: Liu Yi L Signed-off-by: Lu Baolu Reviewed-by: Ashok Raj Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel --- Documentation/admin-guide/kernel-parameters.txt | 12 ++--- drivers/iommu/intel-iommu.c | 64 +++++++------------------ include/linux/intel-iommu.h | 1 + 3 files changed, 24 insertions(+), 53 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 81d1d5a74728..abe9769a9276 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1682,12 +1682,12 @@ By default, super page will be supported if Intel IOMMU has the capability. With this option, super page will not be supported. - ecs_off [Default Off] - By default, extended context tables will be supported if - the hardware advertises that it has support both for the - extended tables themselves, and also PASID support. With - this option set, extended tables will not be used even - on hardware which claims to support them. + sm_off [Default Off] + By default, scalable mode will be supported if the + hardware advertises that it has support for the scalable + mode translation. With this option set, scalable mode + will not be used even on hardware which claims to support + it. tboot_noforce [Default Off] Do not force the Intel IOMMU enabled under tboot. By default, tboot will force Intel IOMMU on, which diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index fdf79baf1d79..2b9784a1887b 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -405,38 +405,16 @@ static int dmar_map_gfx = 1; static int dmar_forcedac; static int intel_iommu_strict; static int intel_iommu_superpage = 1; -static int intel_iommu_ecs = 1; -static int intel_iommu_pasid28; +static int intel_iommu_sm = 1; static int iommu_identity_mapping; #define IDENTMAP_ALL 1 #define IDENTMAP_GFX 2 #define IDENTMAP_AZALIA 4 -/* Broadwell and Skylake have broken ECS support — normal so-called "second - * level" translation of DMA requests-without-PASID doesn't actually happen - * unless you also set the NESTE bit in an extended context-entry. Which of - * course means that SVM doesn't work because it's trying to do nested - * translation of the physical addresses it finds in the process page tables, - * through the IOVA->phys mapping found in the "second level" page tables. - * - * The VT-d specification was retroactively changed to change the definition - * of the capability bits and pretend that Broadwell/Skylake never happened... - * but unfortunately the wrong bit was changed. It's ECS which is broken, but - * for some reason it was the PASID capability bit which was redefined (from - * bit 28 on BDW/SKL to bit 40 in future). - * - * So our test for ECS needs to eschew those implementations which set the old - * PASID capabiity bit 28, since those are the ones on which ECS is broken. - * Unless we are working around the 'pasid28' limitations, that is, by putting - * the device into passthrough mode for normal DMA and thus masking the bug. - */ -#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \ - (intel_iommu_pasid28 || !ecap_broken_pasid(iommu->ecap))) -/* PASID support is thus enabled if ECS is enabled and *either* of the old - * or new capability bits are set. */ -#define pasid_enabled(iommu) (ecs_enabled(iommu) && \ - (ecap_pasid(iommu->ecap) || ecap_broken_pasid(iommu->ecap))) +#define sm_supported(iommu) (intel_iommu_sm && ecap_smts((iommu)->ecap)) +#define pasid_supported(iommu) (sm_supported(iommu) && \ + ecap_pasid((iommu)->ecap)) int intel_iommu_gfx_mapped; EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped); @@ -516,15 +494,9 @@ static int __init intel_iommu_setup(char *str) } else if (!strncmp(str, "sp_off", 6)) { pr_info("Disable supported super page\n"); intel_iommu_superpage = 0; - } else if (!strncmp(str, "ecs_off", 7)) { - printk(KERN_INFO - "Intel-IOMMU: disable extended context table support\n"); - intel_iommu_ecs = 0; - } else if (!strncmp(str, "pasid28", 7)) { - printk(KERN_INFO - "Intel-IOMMU: enable pre-production PASID support\n"); - intel_iommu_pasid28 = 1; - iommu_identity_mapping |= IDENTMAP_GFX; + } else if (!strncmp(str, "sm_off", 6)) { + pr_info("Intel-IOMMU: disable scalable mode support\n"); + intel_iommu_sm = 0; } else if (!strncmp(str, "tboot_noforce", 13)) { printk(KERN_INFO "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n"); @@ -771,7 +743,7 @@ struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus, u64 *entry; entry = &root->lo; - if (ecs_enabled(iommu)) { + if (sm_supported(iommu)) { if (devfn >= 0x80) { devfn -= 0x80; entry = &root->hi; @@ -913,7 +885,7 @@ static void free_context_table(struct intel_iommu *iommu) if (context) free_pgtable_page(context); - if (!ecs_enabled(iommu)) + if (!sm_supported(iommu)) continue; context = iommu_context_addr(iommu, i, 0x80, 0); @@ -1265,8 +1237,6 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) unsigned long flag; addr = virt_to_phys(iommu->root_entry); - if (ecs_enabled(iommu)) - addr |= DMA_RTADDR_RTT; raw_spin_lock_irqsave(&iommu->register_lock, flag); dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr); @@ -1755,7 +1725,7 @@ static void free_dmar_iommu(struct intel_iommu *iommu) free_context_table(iommu); #ifdef CONFIG_INTEL_IOMMU_SVM - if (pasid_enabled(iommu)) { + if (pasid_supported(iommu)) { if (ecap_prs(iommu->ecap)) intel_svm_finish_prq(iommu); intel_svm_exit(iommu); @@ -2464,8 +2434,8 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, dmar_find_matched_atsr_unit(pdev)) info->ats_supported = 1; - if (ecs_enabled(iommu)) { - if (pasid_enabled(iommu)) { + if (sm_supported(iommu)) { + if (pasid_supported(iommu)) { int features = pci_pasid_features(pdev); if (features >= 0) info->pasid_supported = features | 1; @@ -3277,7 +3247,7 @@ static int __init init_dmars(void) * We need to ensure the system pasid table is no bigger * than the smallest supported. */ - if (pasid_enabled(iommu)) { + if (pasid_supported(iommu)) { u32 temp = 2 << ecap_pss(iommu->ecap); intel_pasid_max_id = min_t(u32, temp, @@ -3338,7 +3308,7 @@ static int __init init_dmars(void) if (!ecap_pass_through(iommu->ecap)) hw_pass_through = 0; #ifdef CONFIG_INTEL_IOMMU_SVM - if (pasid_enabled(iommu)) + if (pasid_supported(iommu)) intel_svm_init(iommu); #endif } @@ -3442,7 +3412,7 @@ domains_done: iommu_flush_write_buffer(iommu); #ifdef CONFIG_INTEL_IOMMU_SVM - if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) { + if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) { ret = intel_svm_enable_prq(iommu); if (ret) goto free_iommu; @@ -4331,7 +4301,7 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) goto out; #ifdef CONFIG_INTEL_IOMMU_SVM - if (pasid_enabled(iommu)) + if (pasid_supported(iommu)) intel_svm_init(iommu); #endif @@ -4348,7 +4318,7 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) iommu_flush_write_buffer(iommu); #ifdef CONFIG_INTEL_IOMMU_SVM - if (pasid_enabled(iommu) && ecap_prs(iommu->ecap)) { + if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) { ret = intel_svm_enable_prq(iommu); if (ret) goto disable_iommu; diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index a58bc05d6798..8c9b6063d275 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -177,6 +177,7 @@ * Extended Capability Register */ +#define ecap_smts(e) (((e) >> 43) & 0x1) #define ecap_dit(e) ((e >> 41) & 0x1) #define ecap_pasid(e) ((e >> 40) & 0x1) #define ecap_pss(e) ((e >> 35) & 0x1f) -- cgit v1.2.3 From 0bbeb01a4fafbf8422e5c8882d461d6ac4f71e15 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 10 Dec 2018 09:58:56 +0800 Subject: iommu/vt-d: Manage scalalble mode PASID tables In scalable mode, pasid structure is a two level table with a pasid directory table and a pasid table. Any pasid entry can be identified by a pasid value in below way. 1 9 6 5 0 .-----------------------.-------. | PASID | | '-----------------------'-------' .-------------. | | | | | | | | | | | | | .-----------. | .-------------. | | | |----->| PASID Entry | | | | | '-------------' | | | |Plus | | | .-----------. | | | |---->| DIR Entry |-------->| | | '-----------' '-------------' .---------. |Plus | | | Context | | | | | Entry |------->| | '---------' '-----------' This changes the pasid table APIs to support scalable mode PASID directory and PASID table. It also adds a helper to get the PASID table entry according to the pasid value. Cc: Ashok Raj Cc: Jacob Pan Cc: Kevin Tian Signed-off-by: Sanjay Kumar Signed-off-by: Liu Yi L Signed-off-by: Lu Baolu Reviewed-by: Ashok Raj Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 23 +++++++----- drivers/iommu/intel-pasid.c | 87 +++++++++++++++++++++++++++++++++++++-------- drivers/iommu/intel-pasid.h | 12 +++++-- drivers/iommu/intel-svm.c | 6 +--- 4 files changed, 97 insertions(+), 31 deletions(-) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 2b9784a1887b..5552a1aaf5ea 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -425,21 +425,24 @@ static LIST_HEAD(device_domain_list); /* * Iterate over elements in device_domain_list and call the specified - * callback @fn against each element. This helper should only be used - * in the context where the device_domain_lock has already been holden. + * callback @fn against each element. */ int for_each_device_domain(int (*fn)(struct device_domain_info *info, void *data), void *data) { int ret = 0; + unsigned long flags; struct device_domain_info *info; - assert_spin_locked(&device_domain_lock); + spin_lock_irqsave(&device_domain_lock, flags); list_for_each_entry(info, &device_domain_list, global) { ret = fn(info, data); - if (ret) + if (ret) { + spin_unlock_irqrestore(&device_domain_lock, flags); return ret; + } } + spin_unlock_irqrestore(&device_domain_lock, flags); return 0; } @@ -2481,16 +2484,18 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, list_add(&info->global, &device_domain_list); if (dev) dev->archdata.iommu = info; + spin_unlock_irqrestore(&device_domain_lock, flags); - if (dev && dev_is_pci(dev) && info->pasid_supported) { + /* PASID table is mandatory for a PCI device in scalable mode. */ + if (dev && dev_is_pci(dev) && sm_supported(iommu)) { ret = intel_pasid_alloc_table(dev); if (ret) { - pr_warn("No pasid table for %s, pasid disabled\n", - dev_name(dev)); - info->pasid_supported = 0; + pr_err("PASID table allocation for %s failed\n", + dev_name(dev)); + dmar_remove_one_dev_info(domain, dev); + return NULL; } } - spin_unlock_irqrestore(&device_domain_lock, flags); if (dev && domain_context_mapping(domain, dev)) { pr_err("Domain context map for %s failed\n", dev_name(dev)); diff --git a/drivers/iommu/intel-pasid.c b/drivers/iommu/intel-pasid.c index fe95c9bd4d33..fd3ccc0753b0 100644 --- a/drivers/iommu/intel-pasid.c +++ b/drivers/iommu/intel-pasid.c @@ -123,12 +123,13 @@ int intel_pasid_alloc_table(struct device *dev) struct pasid_table *pasid_table; struct pasid_table_opaque data; struct page *pages; - size_t size, count; + int max_pasid = 0; int ret, order; + int size; + might_sleep(); info = dev->archdata.iommu; - if (WARN_ON(!info || !dev_is_pci(dev) || - !info->pasid_supported || info->pasid_table)) + if (WARN_ON(!info || !dev_is_pci(dev) || info->pasid_table)) return -EINVAL; /* DMA alias device already has a pasid table, use it: */ @@ -138,23 +139,25 @@ int intel_pasid_alloc_table(struct device *dev) if (ret) goto attach_out; - pasid_table = kzalloc(sizeof(*pasid_table), GFP_ATOMIC); + pasid_table = kzalloc(sizeof(*pasid_table), GFP_KERNEL); if (!pasid_table) return -ENOMEM; INIT_LIST_HEAD(&pasid_table->dev); - size = sizeof(struct pasid_entry); - count = min_t(int, pci_max_pasids(to_pci_dev(dev)), intel_pasid_max_id); - order = get_order(size * count); + if (info->pasid_supported) + max_pasid = min_t(int, pci_max_pasids(to_pci_dev(dev)), + intel_pasid_max_id); + + size = max_pasid >> (PASID_PDE_SHIFT - 3); + order = size ? get_order(size) : 0; pages = alloc_pages_node(info->iommu->node, - GFP_ATOMIC | __GFP_ZERO, - order); + GFP_KERNEL | __GFP_ZERO, order); if (!pages) return -ENOMEM; pasid_table->table = page_address(pages); pasid_table->order = order; - pasid_table->max_pasid = count; + pasid_table->max_pasid = 1 << (order + PAGE_SHIFT + 3); attach_out: device_attach_pasid_table(info, pasid_table); @@ -162,14 +165,33 @@ attach_out: return 0; } +/* Get PRESENT bit of a PASID directory entry. */ +static inline bool +pasid_pde_is_present(struct pasid_dir_entry *pde) +{ + return READ_ONCE(pde->val) & PASID_PTE_PRESENT; +} + +/* Get PASID table from a PASID directory entry. */ +static inline struct pasid_entry * +get_pasid_table_from_pde(struct pasid_dir_entry *pde) +{ + if (!pasid_pde_is_present(pde)) + return NULL; + + return phys_to_virt(READ_ONCE(pde->val) & PDE_PFN_MASK); +} + void intel_pasid_free_table(struct device *dev) { struct device_domain_info *info; struct pasid_table *pasid_table; + struct pasid_dir_entry *dir; + struct pasid_entry *table; + int i, max_pde; info = dev->archdata.iommu; - if (!info || !dev_is_pci(dev) || - !info->pasid_supported || !info->pasid_table) + if (!info || !dev_is_pci(dev) || !info->pasid_table) return; pasid_table = info->pasid_table; @@ -178,6 +200,14 @@ void intel_pasid_free_table(struct device *dev) if (!list_empty(&pasid_table->dev)) return; + /* Free scalable mode PASID directory tables: */ + dir = pasid_table->table; + max_pde = pasid_table->max_pasid >> PASID_PDE_SHIFT; + for (i = 0; i < max_pde; i++) { + table = get_pasid_table_from_pde(&dir[i]); + free_pgtable_page(table); + } + free_pages((unsigned long)pasid_table->table, pasid_table->order); kfree(pasid_table); } @@ -206,17 +236,37 @@ int intel_pasid_get_dev_max_id(struct device *dev) struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid) { + struct device_domain_info *info; struct pasid_table *pasid_table; + struct pasid_dir_entry *dir; struct pasid_entry *entries; + int dir_index, index; pasid_table = intel_pasid_get_table(dev); if (WARN_ON(!pasid_table || pasid < 0 || pasid >= intel_pasid_get_dev_max_id(dev))) return NULL; - entries = pasid_table->table; + dir = pasid_table->table; + info = dev->archdata.iommu; + dir_index = pasid >> PASID_PDE_SHIFT; + index = pasid & PASID_PTE_MASK; + + spin_lock(&pasid_lock); + entries = get_pasid_table_from_pde(&dir[dir_index]); + if (!entries) { + entries = alloc_pgtable_page(info->iommu->node); + if (!entries) { + spin_unlock(&pasid_lock); + return NULL; + } + + WRITE_ONCE(dir[dir_index].val, + (u64)virt_to_phys(entries) | PASID_PTE_PRESENT); + } + spin_unlock(&pasid_lock); - return &entries[pasid]; + return &entries[index]; } /* @@ -224,7 +274,14 @@ struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid) */ static inline void pasid_clear_entry(struct pasid_entry *pe) { - WRITE_ONCE(pe->val, 0); + WRITE_ONCE(pe->val[0], 0); + WRITE_ONCE(pe->val[1], 0); + WRITE_ONCE(pe->val[2], 0); + WRITE_ONCE(pe->val[3], 0); + WRITE_ONCE(pe->val[4], 0); + WRITE_ONCE(pe->val[5], 0); + WRITE_ONCE(pe->val[6], 0); + WRITE_ONCE(pe->val[7], 0); } void intel_pasid_clear_entry(struct device *dev, int pasid) diff --git a/drivers/iommu/intel-pasid.h b/drivers/iommu/intel-pasid.h index 1fb5e12b029a..12f480c2bb8b 100644 --- a/drivers/iommu/intel-pasid.h +++ b/drivers/iommu/intel-pasid.h @@ -11,12 +11,20 @@ #define __INTEL_PASID_H #define PASID_MIN 0x1 -#define PASID_MAX 0x20000 +#define PASID_MAX 0x100000 +#define PASID_PTE_MASK 0x3F +#define PASID_PTE_PRESENT 1 +#define PDE_PFN_MASK PAGE_MASK +#define PASID_PDE_SHIFT 6 -struct pasid_entry { +struct pasid_dir_entry { u64 val; }; +struct pasid_entry { + u64 val[8]; +}; + /* The representative of a PASID table */ struct pasid_table { void *table; /* pasid table pointer */ diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index db301efe126d..d6c99935d5d9 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -65,8 +65,6 @@ int intel_svm_init(struct intel_iommu *iommu) order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max); if (ecap_dis(iommu->ecap)) { - /* Just making it explicit... */ - BUILD_BUG_ON(sizeof(struct pasid_entry) != sizeof(struct pasid_state_entry)); pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); if (pages) iommu->pasid_state_table = page_address(pages); @@ -405,9 +403,7 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ pasid_entry_val |= PASID_ENTRY_FLPM_5LP; entry = intel_pasid_get_entry(dev, svm->pasid); - entry->val = pasid_entry_val; - - wmb(); + WRITE_ONCE(entry->val[0], pasid_entry_val); /* * Flush PASID cache when a PASID table entry becomes -- cgit v1.2.3 From 4f2ed183cfebf42b29ed8fe442169de97bc0fe61 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 10 Dec 2018 09:58:57 +0800 Subject: iommu/vt-d: Move page table helpers into header So that they could also be used in other source files. Cc: Ashok Raj Cc: Jacob Pan Cc: Kevin Tian Cc: Liu Yi L Cc: Sanjay Kumar Signed-off-by: Lu Baolu Reviewed-by: Ashok Raj Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 43 ------------------------------------------- include/linux/intel-iommu.h | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 5552a1aaf5ea..d55254abd5ff 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -290,49 +290,6 @@ static inline void context_clear_entry(struct context_entry *context) context->hi = 0; } -/* - * 0: readable - * 1: writable - * 2-6: reserved - * 7: super page - * 8-10: available - * 11: snoop behavior - * 12-63: Host physcial address - */ -struct dma_pte { - u64 val; -}; - -static inline void dma_clear_pte(struct dma_pte *pte) -{ - pte->val = 0; -} - -static inline u64 dma_pte_addr(struct dma_pte *pte) -{ -#ifdef CONFIG_64BIT - return pte->val & VTD_PAGE_MASK; -#else - /* Must have a full atomic 64-bit read */ - return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK; -#endif -} - -static inline bool dma_pte_present(struct dma_pte *pte) -{ - return (pte->val & 3) != 0; -} - -static inline bool dma_pte_superpage(struct dma_pte *pte) -{ - return (pte->val & DMA_PTE_LARGE_PAGE); -} - -static inline int first_pte_in_page(struct dma_pte *pte) -{ - return !((unsigned long)pte & ~VTD_PAGE_MASK); -} - /* * This domain is a statically identity mapping domain. * 1. This domain creats a static 1:1 mapping to all usable memory. diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 8c9b6063d275..b4da61385ebf 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -590,6 +590,49 @@ static inline void __iommu_flush_cache( clflush_cache_range(addr, size); } +/* + * 0: readable + * 1: writable + * 2-6: reserved + * 7: super page + * 8-10: available + * 11: snoop behavior + * 12-63: Host physcial address + */ +struct dma_pte { + u64 val; +}; + +static inline void dma_clear_pte(struct dma_pte *pte) +{ + pte->val = 0; +} + +static inline u64 dma_pte_addr(struct dma_pte *pte) +{ +#ifdef CONFIG_64BIT + return pte->val & VTD_PAGE_MASK; +#else + /* Must have a full atomic 64-bit read */ + return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK; +#endif +} + +static inline bool dma_pte_present(struct dma_pte *pte) +{ + return (pte->val & 3) != 0; +} + +static inline bool dma_pte_superpage(struct dma_pte *pte) +{ + return (pte->val & DMA_PTE_LARGE_PAGE); +} + +static inline int first_pte_in_page(struct dma_pte *pte) +{ + return !((unsigned long)pte & ~VTD_PAGE_MASK); +} + extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev); extern int dmar_find_matched_atsr_unit(struct pci_dev *dev); -- cgit v1.2.3 From 5d308fc1ecf5351418a4f003ccb74dc91b424bd1 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 10 Dec 2018 09:58:58 +0800 Subject: iommu/vt-d: Add 256-bit invalidation descriptor support Intel vt-d spec rev3.0 requires software to use 256-bit descriptors in invalidation queue. As the spec reads in section 6.5.2: Remapping hardware supporting Scalable Mode Translations (ECAP_REG.SMTS=1) allow software to additionally program the width of the descriptors (128-bits or 256-bits) that will be written into the Queue. Software should setup the Invalidation Queue for 256-bit descriptors before progra- mming remapping hardware for scalable-mode translation as 128-bit descriptors are treated as invalid descriptors (see Table 21 in Section 6.5.2.10) in scalable-mode. This patch adds 256-bit invalidation descriptor support if the hardware presents scalable mode capability. Cc: Ashok Raj Cc: Jacob Pan Cc: Kevin Tian Signed-off-by: Sanjay Kumar Signed-off-by: Liu Yi L Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 91 +++++++++++++++++++++++++------------ drivers/iommu/intel-svm.c | 76 ++++++++++++++++++++----------- drivers/iommu/intel_irq_remapping.c | 6 ++- include/linux/intel-iommu.h | 9 +++- 4 files changed, 121 insertions(+), 61 deletions(-) diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index d9c748b6f9e4..9511f9aeb77c 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -1160,6 +1160,7 @@ static int qi_check_fault(struct intel_iommu *iommu, int index) int head, tail; struct q_inval *qi = iommu->qi; int wait_index = (index + 1) % QI_LENGTH; + int shift = qi_shift(iommu); if (qi->desc_status[wait_index] == QI_ABORT) return -EAGAIN; @@ -1173,13 +1174,19 @@ static int qi_check_fault(struct intel_iommu *iommu, int index) */ if (fault & DMA_FSTS_IQE) { head = readl(iommu->reg + DMAR_IQH_REG); - if ((head >> DMAR_IQ_SHIFT) == index) { - pr_err("VT-d detected invalid descriptor: " - "low=%llx, high=%llx\n", - (unsigned long long)qi->desc[index].low, - (unsigned long long)qi->desc[index].high); - memcpy(&qi->desc[index], &qi->desc[wait_index], - sizeof(struct qi_desc)); + if ((head >> shift) == index) { + struct qi_desc *desc = qi->desc + head; + + /* + * desc->qw2 and desc->qw3 are either reserved or + * used by software as private data. We won't print + * out these two qw's for security consideration. + */ + pr_err("VT-d detected invalid descriptor: qw0 = %llx, qw1 = %llx\n", + (unsigned long long)desc->qw0, + (unsigned long long)desc->qw1); + memcpy(desc, qi->desc + (wait_index << shift), + 1 << shift); writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG); return -EINVAL; } @@ -1191,10 +1198,10 @@ static int qi_check_fault(struct intel_iommu *iommu, int index) */ if (fault & DMA_FSTS_ITE) { head = readl(iommu->reg + DMAR_IQH_REG); - head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH; + head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH; head |= 1; tail = readl(iommu->reg + DMAR_IQT_REG); - tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH; + tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH; writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG); @@ -1222,15 +1229,14 @@ int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu) { int rc; struct q_inval *qi = iommu->qi; - struct qi_desc *hw, wait_desc; + int offset, shift, length; + struct qi_desc wait_desc; int wait_index, index; unsigned long flags; if (!qi) return 0; - hw = qi->desc; - restart: rc = 0; @@ -1243,16 +1249,21 @@ restart: index = qi->free_head; wait_index = (index + 1) % QI_LENGTH; + shift = qi_shift(iommu); + length = 1 << shift; qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE; - hw[index] = *desc; - - wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) | + offset = index << shift; + memcpy(qi->desc + offset, desc, length); + wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) | QI_IWD_STATUS_WRITE | QI_IWD_TYPE; - wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]); + wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]); + wait_desc.qw2 = 0; + wait_desc.qw3 = 0; - hw[wait_index] = wait_desc; + offset = wait_index << shift; + memcpy(qi->desc + offset, &wait_desc, length); qi->free_head = (qi->free_head + 2) % QI_LENGTH; qi->free_cnt -= 2; @@ -1261,7 +1272,7 @@ restart: * update the HW tail register indicating the presence of * new descriptors. */ - writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG); + writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG); while (qi->desc_status[wait_index] != QI_DONE) { /* @@ -1298,8 +1309,10 @@ void qi_global_iec(struct intel_iommu *iommu) { struct qi_desc desc; - desc.low = QI_IEC_TYPE; - desc.high = 0; + desc.qw0 = QI_IEC_TYPE; + desc.qw1 = 0; + desc.qw2 = 0; + desc.qw3 = 0; /* should never fail */ qi_submit_sync(&desc, iommu); @@ -1310,9 +1323,11 @@ void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm, { struct qi_desc desc; - desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did) + desc.qw0 = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did) | QI_CC_GRAN(type) | QI_CC_TYPE; - desc.high = 0; + desc.qw1 = 0; + desc.qw2 = 0; + desc.qw3 = 0; qi_submit_sync(&desc, iommu); } @@ -1331,10 +1346,12 @@ void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr, if (cap_read_drain(iommu->cap)) dr = 1; - desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw) + desc.qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw) | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE; - desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih) + desc.qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih) | QI_IOTLB_AM(size_order); + desc.qw2 = 0; + desc.qw3 = 0; qi_submit_sync(&desc, iommu); } @@ -1347,15 +1364,17 @@ void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, if (mask) { WARN_ON_ONCE(addr & ((1ULL << (VTD_PAGE_SHIFT + mask)) - 1)); addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1; - desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE; + desc.qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE; } else - desc.high = QI_DEV_IOTLB_ADDR(addr); + desc.qw1 = QI_DEV_IOTLB_ADDR(addr); if (qdep >= QI_DEV_IOTLB_MAX_INVS) qdep = 0; - desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) | + desc.qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) | QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid); + desc.qw2 = 0; + desc.qw3 = 0; qi_submit_sync(&desc, iommu); } @@ -1403,16 +1422,24 @@ static void __dmar_enable_qi(struct intel_iommu *iommu) u32 sts; unsigned long flags; struct q_inval *qi = iommu->qi; + u64 val = virt_to_phys(qi->desc); qi->free_head = qi->free_tail = 0; qi->free_cnt = QI_LENGTH; + /* + * Set DW=1 and QS=1 in IQA_REG when Scalable Mode capability + * is present. + */ + if (ecap_smts(iommu->ecap)) + val |= (1 << 11) | 1; + raw_spin_lock_irqsave(&iommu->register_lock, flags); /* write zero to the tail reg */ writel(0, iommu->reg + DMAR_IQT_REG); - dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc)); + dmar_writeq(iommu->reg + DMAR_IQA_REG, val); iommu->gcmd |= DMA_GCMD_QIE; writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); @@ -1448,8 +1475,12 @@ int dmar_enable_qi(struct intel_iommu *iommu) qi = iommu->qi; - - desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0); + /* + * Need two pages to accommodate 256 descriptors of 256 bits each + * if the remapping hardware supports scalable mode translation. + */ + desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, + !!ecap_smts(iommu->ecap)); if (!desc_page) { kfree(qi); iommu->qi = NULL; diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index d6c99935d5d9..b7f1d12e24b0 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -161,27 +161,40 @@ static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_d * because that's the only option the hardware gives us. Despite * the fact that they are actually only accessible through one. */ if (gl) - desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) | - QI_EIOTLB_GRAN(QI_GRAN_ALL_ALL) | QI_EIOTLB_TYPE; + desc.qw0 = QI_EIOTLB_PASID(svm->pasid) | + QI_EIOTLB_DID(sdev->did) | + QI_EIOTLB_GRAN(QI_GRAN_ALL_ALL) | + QI_EIOTLB_TYPE; else - desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) | - QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | QI_EIOTLB_TYPE; - desc.high = 0; + desc.qw0 = QI_EIOTLB_PASID(svm->pasid) | + QI_EIOTLB_DID(sdev->did) | + QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | + QI_EIOTLB_TYPE; + desc.qw1 = 0; } else { int mask = ilog2(__roundup_pow_of_two(pages)); - desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) | - QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | QI_EIOTLB_TYPE; - desc.high = QI_EIOTLB_ADDR(address) | QI_EIOTLB_GL(gl) | - QI_EIOTLB_IH(ih) | QI_EIOTLB_AM(mask); + desc.qw0 = QI_EIOTLB_PASID(svm->pasid) | + QI_EIOTLB_DID(sdev->did) | + QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | + QI_EIOTLB_TYPE; + desc.qw1 = QI_EIOTLB_ADDR(address) | + QI_EIOTLB_GL(gl) | + QI_EIOTLB_IH(ih) | + QI_EIOTLB_AM(mask); } + desc.qw2 = 0; + desc.qw3 = 0; qi_submit_sync(&desc, svm->iommu); if (sdev->dev_iotlb) { - desc.low = QI_DEV_EIOTLB_PASID(svm->pasid) | QI_DEV_EIOTLB_SID(sdev->sid) | - QI_DEV_EIOTLB_QDEP(sdev->qdep) | QI_DEIOTLB_TYPE; + desc.qw0 = QI_DEV_EIOTLB_PASID(svm->pasid) | + QI_DEV_EIOTLB_SID(sdev->sid) | + QI_DEV_EIOTLB_QDEP(sdev->qdep) | + QI_DEIOTLB_TYPE; if (pages == -1) { - desc.high = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) | QI_DEV_EIOTLB_SIZE; + desc.qw1 = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) | + QI_DEV_EIOTLB_SIZE; } else if (pages > 1) { /* The least significant zero bit indicates the size. So, * for example, an "address" value of 0x12345f000 will @@ -189,10 +202,13 @@ static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_d unsigned long last = address + ((unsigned long)(pages - 1) << VTD_PAGE_SHIFT); unsigned long mask = __rounddown_pow_of_two(address ^ last); - desc.high = QI_DEV_EIOTLB_ADDR((address & ~mask) | (mask - 1)) | QI_DEV_EIOTLB_SIZE; + desc.qw1 = QI_DEV_EIOTLB_ADDR((address & ~mask) | + (mask - 1)) | QI_DEV_EIOTLB_SIZE; } else { - desc.high = QI_DEV_EIOTLB_ADDR(address); + desc.qw1 = QI_DEV_EIOTLB_ADDR(address); } + desc.qw2 = 0; + desc.qw3 = 0; qi_submit_sync(&desc, svm->iommu); } } @@ -237,8 +253,11 @@ static void intel_flush_pasid_dev(struct intel_svm *svm, struct intel_svm_dev *s { struct qi_desc desc; - desc.high = 0; - desc.low = QI_PC_TYPE | QI_PC_DID(sdev->did) | QI_PC_PASID_SEL | QI_PC_PASID(pasid); + desc.qw0 = QI_PC_TYPE | QI_PC_DID(sdev->did) | + QI_PC_PASID_SEL | QI_PC_PASID(pasid); + desc.qw1 = 0; + desc.qw2 = 0; + desc.qw3 = 0; qi_submit_sync(&desc, svm->iommu); } @@ -667,24 +686,27 @@ static irqreturn_t prq_event_thread(int irq, void *d) no_pasid: if (req->lpig) { /* Page Group Response */ - resp.low = QI_PGRP_PASID(req->pasid) | + resp.qw0 = QI_PGRP_PASID(req->pasid) | QI_PGRP_DID((req->bus << 8) | req->devfn) | QI_PGRP_PASID_P(req->pasid_present) | QI_PGRP_RESP_TYPE; - resp.high = QI_PGRP_IDX(req->prg_index) | - QI_PGRP_PRIV(req->private) | QI_PGRP_RESP_CODE(result); - - qi_submit_sync(&resp, iommu); + resp.qw1 = QI_PGRP_IDX(req->prg_index) | + QI_PGRP_PRIV(req->private) | + QI_PGRP_RESP_CODE(result); } else if (req->srr) { /* Page Stream Response */ - resp.low = QI_PSTRM_IDX(req->prg_index) | - QI_PSTRM_PRIV(req->private) | QI_PSTRM_BUS(req->bus) | - QI_PSTRM_PASID(req->pasid) | QI_PSTRM_RESP_TYPE; - resp.high = QI_PSTRM_ADDR(address) | QI_PSTRM_DEVFN(req->devfn) | + resp.qw0 = QI_PSTRM_IDX(req->prg_index) | + QI_PSTRM_PRIV(req->private) | + QI_PSTRM_BUS(req->bus) | + QI_PSTRM_PASID(req->pasid) | + QI_PSTRM_RESP_TYPE; + resp.qw1 = QI_PSTRM_ADDR(address) | + QI_PSTRM_DEVFN(req->devfn) | QI_PSTRM_RESP_CODE(result); - - qi_submit_sync(&resp, iommu); } + resp.qw2 = 0; + resp.qw3 = 0; + qi_submit_sync(&resp, iommu); head = (head + sizeof(*req)) & PRQ_RING_MASK; } diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index c2d6c11431de..24d45b07f425 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -145,9 +145,11 @@ static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask) { struct qi_desc desc; - desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask) + desc.qw0 = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask) | QI_IEC_SELECTIVE; - desc.high = 0; + desc.qw1 = 0; + desc.qw2 = 0; + desc.qw3 = 0; return qi_submit_sync(&desc, iommu); } diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index b4da61385ebf..08ff588a4df7 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -401,13 +401,18 @@ enum { #define QI_GRAN_NONG_PASID 2 #define QI_GRAN_PSI_PASID 3 +#define qi_shift(iommu) (DMAR_IQ_SHIFT + !!ecap_smts((iommu)->ecap)) + struct qi_desc { - u64 low, high; + u64 qw0; + u64 qw1; + u64 qw2; + u64 qw3; }; struct q_inval { raw_spinlock_t q_lock; - struct qi_desc *desc; /* invalidation queue */ + void *desc; /* invalidation queue */ int *desc_status; /* desc status */ int free_head; /* first free entry */ int free_tail; /* last free entry */ -- cgit v1.2.3 From 3b33d4ab3217337f648f7ff3e57e755e6444c748 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 10 Dec 2018 09:58:59 +0800 Subject: iommu/vt-d: Reserve a domain id for FL and PT modes Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid entry for first-level or pass-through translation should be programmed with a domain id different from those used for second-level or nested translation. It is recommended that software could use a same domain id for all first-only and pass-through translations. This reserves a domain id for first-level and pass-through translations. Cc: Ashok Raj Cc: Jacob Pan Cc: Kevin Tian Cc: Liu Yi L Cc: Sanjay Kumar Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 10 ++++++++++ drivers/iommu/intel-pasid.h | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index d55254abd5ff..9818aaf2d0f7 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1621,6 +1621,16 @@ static int iommu_init_domains(struct intel_iommu *iommu) */ set_bit(0, iommu->domain_ids); + /* + * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid + * entry for first-level or pass-through translation modes should + * be programmed with a domain id different from those used for + * second-level or nested translation. We reserve a domain id for + * this purpose. + */ + if (sm_supported(iommu)) + set_bit(FLPT_DEFAULT_DID, iommu->domain_ids); + return 0; } diff --git a/drivers/iommu/intel-pasid.h b/drivers/iommu/intel-pasid.h index 12f480c2bb8b..03c1612d173c 100644 --- a/drivers/iommu/intel-pasid.h +++ b/drivers/iommu/intel-pasid.h @@ -17,6 +17,12 @@ #define PDE_PFN_MASK PAGE_MASK #define PASID_PDE_SHIFT 6 +/* + * Domain ID reserved for pasid entries programmed for first-level + * only and pass-through transfer modes. + */ +#define FLPT_DEFAULT_DID 1 + struct pasid_dir_entry { u64 val; }; -- cgit v1.2.3 From 6f7db75e1c469057fe7588ed959328ead771ccc7 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 10 Dec 2018 09:59:00 +0800 Subject: iommu/vt-d: Add second level page table interface This adds the interfaces to setup or tear down the structures for second level page table translations. This includes types of second level only translation and pass through. Cc: Ashok Raj Cc: Jacob Pan Cc: Kevin Tian Signed-off-by: Sanjay Kumar Signed-off-by: Liu Yi L Signed-off-by: Lu Baolu Reviewed-by: Ashok Raj Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 2 +- drivers/iommu/intel-pasid.c | 280 ++++++++++++++++++++++++++++++++++++++++++++ drivers/iommu/intel-pasid.h | 8 ++ include/linux/intel-iommu.h | 3 + 4 files changed, 292 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 9818aaf2d0f7..f2976a3f1d67 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1210,7 +1210,7 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) raw_spin_unlock_irqrestore(&iommu->register_lock, flag); } -static void iommu_flush_write_buffer(struct intel_iommu *iommu) +void iommu_flush_write_buffer(struct intel_iommu *iommu) { u32 val; unsigned long flag; diff --git a/drivers/iommu/intel-pasid.c b/drivers/iommu/intel-pasid.c index fd3ccc0753b0..6d2b2e87e6fc 100644 --- a/drivers/iommu/intel-pasid.c +++ b/drivers/iommu/intel-pasid.c @@ -9,6 +9,7 @@ #define pr_fmt(fmt) "DMAR: " fmt +#include #include #include #include @@ -294,3 +295,282 @@ void intel_pasid_clear_entry(struct device *dev, int pasid) pasid_clear_entry(pe); } + +static inline void pasid_set_bits(u64 *ptr, u64 mask, u64 bits) +{ + u64 old; + + old = READ_ONCE(*ptr); + WRITE_ONCE(*ptr, (old & ~mask) | bits); +} + +/* + * Setup the DID(Domain Identifier) field (Bit 64~79) of scalable mode + * PASID entry. + */ +static inline void +pasid_set_domain_id(struct pasid_entry *pe, u64 value) +{ + pasid_set_bits(&pe->val[1], GENMASK_ULL(15, 0), value); +} + +/* + * Get domain ID value of a scalable mode PASID entry. + */ +static inline u16 +pasid_get_domain_id(struct pasid_entry *pe) +{ + return (u16)(READ_ONCE(pe->val[1]) & GENMASK_ULL(15, 0)); +} + +/* + * Setup the SLPTPTR(Second Level Page Table Pointer) field (Bit 12~63) + * of a scalable mode PASID entry. + */ +static inline void +pasid_set_slptr(struct pasid_entry *pe, u64 value) +{ + pasid_set_bits(&pe->val[0], VTD_PAGE_MASK, value); +} + +/* + * Setup the AW(Address Width) field (Bit 2~4) of a scalable mode PASID + * entry. + */ +static inline void +pasid_set_address_width(struct pasid_entry *pe, u64 value) +{ + pasid_set_bits(&pe->val[0], GENMASK_ULL(4, 2), value << 2); +} + +/* + * Setup the PGTT(PASID Granular Translation Type) field (Bit 6~8) + * of a scalable mode PASID entry. + */ +static inline void +pasid_set_translation_type(struct pasid_entry *pe, u64 value) +{ + pasid_set_bits(&pe->val[0], GENMASK_ULL(8, 6), value << 6); +} + +/* + * Enable fault processing by clearing the FPD(Fault Processing + * Disable) field (Bit 1) of a scalable mode PASID entry. + */ +static inline void pasid_set_fault_enable(struct pasid_entry *pe) +{ + pasid_set_bits(&pe->val[0], 1 << 1, 0); +} + +/* + * Setup the SRE(Supervisor Request Enable) field (Bit 128) of a + * scalable mode PASID entry. + */ +static inline void pasid_set_sre(struct pasid_entry *pe) +{ + pasid_set_bits(&pe->val[2], 1 << 0, 1); +} + +/* + * Setup the P(Present) field (Bit 0) of a scalable mode PASID + * entry. + */ +static inline void pasid_set_present(struct pasid_entry *pe) +{ + pasid_set_bits(&pe->val[0], 1 << 0, 1); +} + +/* + * Setup Page Walk Snoop bit (Bit 87) of a scalable mode PASID + * entry. + */ +static inline void pasid_set_page_snoop(struct pasid_entry *pe, bool value) +{ + pasid_set_bits(&pe->val[1], 1 << 23, value); +} + +static void +pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu, + u16 did, int pasid) +{ + struct qi_desc desc; + + desc.qw0 = QI_PC_DID(did) | QI_PC_PASID_SEL | QI_PC_PASID(pasid); + desc.qw1 = 0; + desc.qw2 = 0; + desc.qw3 = 0; + + qi_submit_sync(&desc, iommu); +} + +static void +iotlb_invalidation_with_pasid(struct intel_iommu *iommu, u16 did, u32 pasid) +{ + struct qi_desc desc; + + desc.qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) | + QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | QI_EIOTLB_TYPE; + desc.qw1 = 0; + desc.qw2 = 0; + desc.qw3 = 0; + + qi_submit_sync(&desc, iommu); +} + +static void +devtlb_invalidation_with_pasid(struct intel_iommu *iommu, + struct device *dev, int pasid) +{ + struct device_domain_info *info; + u16 sid, qdep, pfsid; + + info = dev->archdata.iommu; + if (!info || !info->ats_enabled) + return; + + sid = info->bus << 8 | info->devfn; + qdep = info->ats_qdep; + pfsid = info->pfsid; + + qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 - VTD_PAGE_SHIFT); +} + +void intel_pasid_tear_down_entry(struct intel_iommu *iommu, + struct device *dev, int pasid) +{ + struct pasid_entry *pte; + u16 did; + + pte = intel_pasid_get_entry(dev, pasid); + if (WARN_ON(!pte)) + return; + + intel_pasid_clear_entry(dev, pasid); + did = pasid_get_domain_id(pte); + + if (!ecap_coherent(iommu->ecap)) + clflush_cache_range(pte, sizeof(*pte)); + + pasid_cache_invalidation_with_pasid(iommu, did, pasid); + iotlb_invalidation_with_pasid(iommu, did, pasid); + + /* Device IOTLB doesn't need to be flushed in caching mode. */ + if (!cap_caching_mode(iommu->cap)) + devtlb_invalidation_with_pasid(iommu, dev, pasid); +} + +/* + * Set up the scalable mode pasid entry for second only translation type. + */ +int intel_pasid_setup_second_level(struct intel_iommu *iommu, + struct dmar_domain *domain, + struct device *dev, int pasid) +{ + struct pasid_entry *pte; + struct dma_pte *pgd; + u64 pgd_val; + int agaw; + u16 did; + + /* + * If hardware advertises no support for second level + * translation, return directly. + */ + if (!ecap_slts(iommu->ecap)) { + pr_err("No second level translation support on %s\n", + iommu->name); + return -EINVAL; + } + + /* + * Skip top levels of page tables for iommu which has less agaw + * than default. Unnecessary for PT mode. + */ + pgd = domain->pgd; + for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) { + pgd = phys_to_virt(dma_pte_addr(pgd)); + if (!dma_pte_present(pgd)) { + dev_err(dev, "Invalid domain page table\n"); + return -EINVAL; + } + } + + pgd_val = virt_to_phys(pgd); + did = domain->iommu_did[iommu->seq_id]; + + pte = intel_pasid_get_entry(dev, pasid); + if (!pte) { + dev_err(dev, "Failed to get pasid entry of PASID %d\n", pasid); + return -ENODEV; + } + + pasid_clear_entry(pte); + pasid_set_domain_id(pte, did); + pasid_set_slptr(pte, pgd_val); + pasid_set_address_width(pte, agaw); + pasid_set_translation_type(pte, 2); + pasid_set_fault_enable(pte); + pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); + + /* + * Since it is a second level only translation setup, we should + * set SRE bit as well (addresses are expected to be GPAs). + */ + pasid_set_sre(pte); + pasid_set_present(pte); + + if (!ecap_coherent(iommu->ecap)) + clflush_cache_range(pte, sizeof(*pte)); + + if (cap_caching_mode(iommu->cap)) { + pasid_cache_invalidation_with_pasid(iommu, did, pasid); + iotlb_invalidation_with_pasid(iommu, did, pasid); + } else { + iommu_flush_write_buffer(iommu); + } + + return 0; +} + +/* + * Set up the scalable mode pasid entry for passthrough translation type. + */ +int intel_pasid_setup_pass_through(struct intel_iommu *iommu, + struct dmar_domain *domain, + struct device *dev, int pasid) +{ + u16 did = FLPT_DEFAULT_DID; + struct pasid_entry *pte; + + pte = intel_pasid_get_entry(dev, pasid); + if (!pte) { + dev_err(dev, "Failed to get pasid entry of PASID %d\n", pasid); + return -ENODEV; + } + + pasid_clear_entry(pte); + pasid_set_domain_id(pte, did); + pasid_set_address_width(pte, iommu->agaw); + pasid_set_translation_type(pte, 4); + pasid_set_fault_enable(pte); + pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); + + /* + * We should set SRE bit as well since the addresses are expected + * to be GPAs. + */ + pasid_set_sre(pte); + pasid_set_present(pte); + + if (!ecap_coherent(iommu->ecap)) + clflush_cache_range(pte, sizeof(*pte)); + + if (cap_caching_mode(iommu->cap)) { + pasid_cache_invalidation_with_pasid(iommu, did, pasid); + iotlb_invalidation_with_pasid(iommu, did, pasid); + } else { + iommu_flush_write_buffer(iommu); + } + + return 0; +} diff --git a/drivers/iommu/intel-pasid.h b/drivers/iommu/intel-pasid.h index 03c1612d173c..3c70522091d3 100644 --- a/drivers/iommu/intel-pasid.h +++ b/drivers/iommu/intel-pasid.h @@ -49,5 +49,13 @@ struct pasid_table *intel_pasid_get_table(struct device *dev); int intel_pasid_get_dev_max_id(struct device *dev); struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid); void intel_pasid_clear_entry(struct device *dev, int pasid); +int intel_pasid_setup_second_level(struct intel_iommu *iommu, + struct dmar_domain *domain, + struct device *dev, int pasid); +int intel_pasid_setup_pass_through(struct intel_iommu *iommu, + struct dmar_domain *domain, + struct device *dev, int pasid); +void intel_pasid_tear_down_entry(struct intel_iommu *iommu, + struct device *dev, int pasid); #endif /* __INTEL_PASID_H */ diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 08ff588a4df7..cb3ebda47fa7 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -177,6 +177,8 @@ * Extended Capability Register */ +#define ecap_smpwc(e) (((e) >> 48) & 0x1) +#define ecap_slts(e) (((e) >> 46) & 0x1) #define ecap_smts(e) (((e) >> 43) & 0x1) #define ecap_dit(e) ((e >> 41) & 0x1) #define ecap_pasid(e) ((e >> 40) & 0x1) @@ -662,6 +664,7 @@ void free_pgtable_page(void *vaddr); struct intel_iommu *domain_get_iommu(struct dmar_domain *domain); int for_each_device_domain(int (*fn)(struct device_domain_info *info, void *data), void *data); +void iommu_flush_write_buffer(struct intel_iommu *iommu); #ifdef CONFIG_INTEL_IOMMU_SVM int intel_svm_init(struct intel_iommu *iommu); -- cgit v1.2.3 From ef848b7e5a6a0ef5b10640debce790e12717375b Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 10 Dec 2018 09:59:01 +0800 Subject: iommu/vt-d: Setup pasid entry for RID2PASID support when the scalable mode is enabled, there is no second level page translation pointer in the context entry any more (for DMA request without PASID). Instead, a new RID2PASID field is introduced in the context entry. Software can choose any PASID value to set RID2PASID and then setup the translation in the corresponding PASID entry. Upon receiving a DMA request without PASID, hardware will firstly look at this RID2PASID field and then treat this request as a request with a pasid value specified in RID2PASID field. Though software is allowed to use any PASID for the RID2PASID, we will always use the PASID 0 as a sort of design decision. Cc: Ashok Raj Cc: Jacob Pan Cc: Kevin Tian Signed-off-by: Sanjay Kumar Signed-off-by: Liu Yi L Signed-off-by: Lu Baolu Reviewed-by: Ashok Raj Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 20 ++++++++++++++++++++ drivers/iommu/intel-pasid.h | 1 + 2 files changed, 21 insertions(+) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index f2976a3f1d67..620342bffc0b 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2462,6 +2462,22 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, dmar_remove_one_dev_info(domain, dev); return NULL; } + + /* Setup the PASID entry for requests without PASID: */ + spin_lock(&iommu->lock); + if (hw_pass_through && domain_type_is_si(domain)) + ret = intel_pasid_setup_pass_through(iommu, domain, + dev, PASID_RID2PASID); + else + ret = intel_pasid_setup_second_level(iommu, domain, + dev, PASID_RID2PASID); + spin_unlock(&iommu->lock); + if (ret) { + pr_err("Setup RID2PASID for %s failed\n", + dev_name(dev)); + dmar_remove_one_dev_info(domain, dev); + return NULL; + } } if (dev && domain_context_mapping(domain, dev)) { @@ -4825,6 +4841,10 @@ static void __dmar_remove_one_dev_info(struct device_domain_info *info) iommu = info->iommu; if (info->dev) { + if (dev_is_pci(info->dev) && sm_supported(iommu)) + intel_pasid_tear_down_entry(iommu, info->dev, + PASID_RID2PASID); + iommu_disable_dev_iotlb(info); domain_context_clear(iommu, info->dev); intel_pasid_free_table(info->dev); diff --git a/drivers/iommu/intel-pasid.h b/drivers/iommu/intel-pasid.h index 3c70522091d3..d6f4fead4491 100644 --- a/drivers/iommu/intel-pasid.h +++ b/drivers/iommu/intel-pasid.h @@ -10,6 +10,7 @@ #ifndef __INTEL_PASID_H #define __INTEL_PASID_H +#define PASID_RID2PASID 0x0 #define PASID_MIN 0x1 #define PASID_MAX 0x100000 #define PASID_PTE_MASK 0x3F -- cgit v1.2.3 From ca6e322d60498087a160fad1c4ea8765ff2d21e7 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 10 Dec 2018 09:59:02 +0800 Subject: iommu/vt-d: Pass pasid table to context mapping So that the pasid related info, such as the pasid table and the maximum of pasid could be used during setting up scalable mode context. Cc: Ashok Raj Cc: Jacob Pan Cc: Kevin Tian Cc: Liu Yi L Cc: Sanjay Kumar Signed-off-by: Lu Baolu Reviewed-by: Ashok Raj Reviewed-by: Kevin Tian Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 620342bffc0b..13c3c2dd0459 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1920,6 +1920,7 @@ static void domain_exit(struct dmar_domain *domain) static int domain_context_mapping_one(struct dmar_domain *domain, struct intel_iommu *iommu, + struct pasid_table *table, u8 bus, u8 devfn) { u16 did = domain->iommu_did[iommu->seq_id]; @@ -2042,6 +2043,7 @@ out_unlock: struct domain_context_mapping_data { struct dmar_domain *domain; struct intel_iommu *iommu; + struct pasid_table *table; }; static int domain_context_mapping_cb(struct pci_dev *pdev, @@ -2050,25 +2052,31 @@ static int domain_context_mapping_cb(struct pci_dev *pdev, struct domain_context_mapping_data *data = opaque; return domain_context_mapping_one(data->domain, data->iommu, - PCI_BUS_NUM(alias), alias & 0xff); + data->table, PCI_BUS_NUM(alias), + alias & 0xff); } static int domain_context_mapping(struct dmar_domain *domain, struct device *dev) { + struct domain_context_mapping_data data; + struct pasid_table *table; struct intel_iommu *iommu; u8 bus, devfn; - struct domain_context_mapping_data data; iommu = device_to_iommu(dev, &bus, &devfn); if (!iommu) return -ENODEV; + table = intel_pasid_get_table(dev); + if (!dev_is_pci(dev)) - return domain_context_mapping_one(domain, iommu, bus, devfn); + return domain_context_mapping_one(domain, iommu, table, + bus, devfn); data.domain = domain; data.iommu = iommu; + data.table = table; return pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_mapping_cb, &data); -- cgit v1.2.3 From 7373a8cc381978cfafa4b0285cdd935682f1b2d2 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 10 Dec 2018 09:59:03 +0800 Subject: iommu/vt-d: Setup context and enable RID2PASID support This patch enables the translation for requests without PASID in the scalable mode by setting up the root and context entries. Cc: Ashok Raj Cc: Jacob Pan Cc: Kevin Tian Signed-off-by: Sanjay Kumar Signed-off-by: Liu Yi L Signed-off-by: Lu Baolu Reviewed-by: Ashok Raj Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 136 ++++++++++++++++++++++++++++++++++---------- drivers/iommu/intel-pasid.h | 1 + include/linux/intel-iommu.h | 1 + 3 files changed, 108 insertions(+), 30 deletions(-) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 13c3c2dd0459..21a6853290cc 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1197,6 +1197,8 @@ static void iommu_set_root_entry(struct intel_iommu *iommu) unsigned long flag; addr = virt_to_phys(iommu->root_entry); + if (sm_supported(iommu)) + addr |= DMA_RTADDR_SMT; raw_spin_lock_irqsave(&iommu->register_lock, flag); dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr); @@ -1918,6 +1920,56 @@ static void domain_exit(struct dmar_domain *domain) free_domain_mem(domain); } +/* + * Get the PASID directory size for scalable mode context entry. + * Value of X in the PDTS field of a scalable mode context entry + * indicates PASID directory with 2^(X + 7) entries. + */ +static inline unsigned long context_get_sm_pds(struct pasid_table *table) +{ + int pds, max_pde; + + max_pde = table->max_pasid >> PASID_PDE_SHIFT; + pds = find_first_bit((unsigned long *)&max_pde, MAX_NR_PASID_BITS); + if (pds < 7) + return 0; + + return pds - 7; +} + +/* + * Set the RID_PASID field of a scalable mode context entry. The + * IOMMU hardware will use the PASID value set in this field for + * DMA translations of DMA requests without PASID. + */ +static inline void +context_set_sm_rid2pasid(struct context_entry *context, unsigned long pasid) +{ + context->hi |= pasid & ((1 << 20) - 1); + context->hi |= (1 << 20); +} + +/* + * Set the DTE(Device-TLB Enable) field of a scalable mode context + * entry. + */ +static inline void context_set_sm_dte(struct context_entry *context) +{ + context->lo |= (1 << 2); +} + +/* + * Set the PRE(Page Request Enable) field of a scalable mode context + * entry. + */ +static inline void context_set_sm_pre(struct context_entry *context) +{ + context->lo |= (1 << 4); +} + +/* Convert value to context PASID directory size field coding. */ +#define context_pdts(pds) (((pds) & 0x7) << 9) + static int domain_context_mapping_one(struct dmar_domain *domain, struct intel_iommu *iommu, struct pasid_table *table, @@ -1928,8 +1980,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, struct device_domain_info *info = NULL; struct context_entry *context; unsigned long flags; - struct dma_pte *pgd; - int ret, agaw; + int ret; WARN_ON(did == 0); @@ -1975,41 +2026,67 @@ static int domain_context_mapping_one(struct dmar_domain *domain, } } - pgd = domain->pgd; - context_clear_entry(context); - context_set_domain_id(context, did); - /* - * Skip top levels of page tables for iommu which has less agaw - * than default. Unnecessary for PT mode. - */ - if (translation != CONTEXT_TT_PASS_THROUGH) { - for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) { - ret = -ENOMEM; - pgd = phys_to_virt(dma_pte_addr(pgd)); - if (!dma_pte_present(pgd)) - goto out_unlock; - } + if (sm_supported(iommu)) { + unsigned long pds; - info = iommu_support_dev_iotlb(domain, iommu, bus, devfn); - if (info && info->ats_supported) - translation = CONTEXT_TT_DEV_IOTLB; - else - translation = CONTEXT_TT_MULTI_LEVEL; + WARN_ON(!table); + + /* Setup the PASID DIR pointer: */ + pds = context_get_sm_pds(table); + context->lo = (u64)virt_to_phys(table->table) | + context_pdts(pds); + + /* Setup the RID_PASID field: */ + context_set_sm_rid2pasid(context, PASID_RID2PASID); - context_set_address_root(context, virt_to_phys(pgd)); - context_set_address_width(context, agaw); - } else { /* - * In pass through mode, AW must be programmed to - * indicate the largest AGAW value supported by - * hardware. And ASR is ignored by hardware. + * Setup the Device-TLB enable bit and Page request + * Enable bit: */ - context_set_address_width(context, iommu->msagaw); + info = iommu_support_dev_iotlb(domain, iommu, bus, devfn); + if (info && info->ats_supported) + context_set_sm_dte(context); + if (info && info->pri_supported) + context_set_sm_pre(context); + } else { + struct dma_pte *pgd = domain->pgd; + int agaw; + + context_set_domain_id(context, did); + context_set_translation_type(context, translation); + + if (translation != CONTEXT_TT_PASS_THROUGH) { + /* + * Skip top levels of page tables for iommu which has + * less agaw than default. Unnecessary for PT mode. + */ + for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) { + ret = -ENOMEM; + pgd = phys_to_virt(dma_pte_addr(pgd)); + if (!dma_pte_present(pgd)) + goto out_unlock; + } + + info = iommu_support_dev_iotlb(domain, iommu, bus, devfn); + if (info && info->ats_supported) + translation = CONTEXT_TT_DEV_IOTLB; + else + translation = CONTEXT_TT_MULTI_LEVEL; + + context_set_address_root(context, virt_to_phys(pgd)); + context_set_address_width(context, agaw); + } else { + /* + * In pass through mode, AW must be programmed to + * indicate the largest AGAW value supported by + * hardware. And ASR is ignored by hardware. + */ + context_set_address_width(context, iommu->msagaw); + } } - context_set_translation_type(context, translation); context_set_fault_enable(context); context_set_present(context); domain_flush_cache(domain, context, sizeof(*context)); @@ -5180,7 +5257,6 @@ static void intel_iommu_put_resv_regions(struct device *dev, } #ifdef CONFIG_INTEL_IOMMU_SVM -#define MAX_NR_PASID_BITS (20) static inline unsigned long intel_iommu_get_pts(struct device *dev) { int pts, max_pasid; diff --git a/drivers/iommu/intel-pasid.h b/drivers/iommu/intel-pasid.h index d6f4fead4491..55bb8715329d 100644 --- a/drivers/iommu/intel-pasid.h +++ b/drivers/iommu/intel-pasid.h @@ -17,6 +17,7 @@ #define PASID_PTE_PRESENT 1 #define PDE_PFN_MASK PAGE_MASK #define PASID_PDE_SHIFT 6 +#define MAX_NR_PASID_BITS 20 /* * Domain ID reserved for pasid entries programmed for first-level diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index cb3ebda47fa7..5fdd33ed2cce 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -258,6 +258,7 @@ /* DMA_RTADDR_REG */ #define DMA_RTADDR_RTT (((u64)1) << 11) +#define DMA_RTADDR_SMT (((u64)1) << 10) /* CCMD_REG */ #define DMA_CCMD_ICC (((u64)1) << 63) -- cgit v1.2.3 From 437f35e1cd4c8d043633bb72f4260369af68fbf7 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 10 Dec 2018 09:59:04 +0800 Subject: iommu/vt-d: Add first level page table interface This adds an interface to setup the PASID entries for first level page table translation. Cc: Ashok Raj Cc: Jacob Pan Cc: Kevin Tian Signed-off-by: Sanjay Kumar Signed-off-by: Liu Yi L Signed-off-by: Lu Baolu Reviewed-by: Ashok Raj Signed-off-by: Joerg Roedel --- drivers/iommu/intel-pasid.c | 80 +++++++++++++++++++++++++++++++++++++++++++++ drivers/iommu/intel-pasid.h | 11 +++++++ include/linux/intel-iommu.h | 1 + 3 files changed, 92 insertions(+) diff --git a/drivers/iommu/intel-pasid.c b/drivers/iommu/intel-pasid.c index 6d2b2e87e6fc..c3dcf4dc2496 100644 --- a/drivers/iommu/intel-pasid.c +++ b/drivers/iommu/intel-pasid.c @@ -10,6 +10,7 @@ #define pr_fmt(fmt) "DMAR: " fmt #include +#include #include #include #include @@ -389,6 +390,26 @@ static inline void pasid_set_page_snoop(struct pasid_entry *pe, bool value) pasid_set_bits(&pe->val[1], 1 << 23, value); } +/* + * Setup the First Level Page table Pointer field (Bit 140~191) + * of a scalable mode PASID entry. + */ +static inline void +pasid_set_flptr(struct pasid_entry *pe, u64 value) +{ + pasid_set_bits(&pe->val[2], VTD_PAGE_MASK, value); +} + +/* + * Setup the First Level Paging Mode field (Bit 130~131) of a + * scalable mode PASID entry. + */ +static inline void +pasid_set_flpm(struct pasid_entry *pe, u64 value) +{ + pasid_set_bits(&pe->val[2], GENMASK_ULL(3, 2), value << 2); +} + static void pasid_cache_invalidation_with_pasid(struct intel_iommu *iommu, u16 did, int pasid) @@ -459,6 +480,65 @@ void intel_pasid_tear_down_entry(struct intel_iommu *iommu, devtlb_invalidation_with_pasid(iommu, dev, pasid); } +/* + * Set up the scalable mode pasid table entry for first only + * translation type. + */ +int intel_pasid_setup_first_level(struct intel_iommu *iommu, + struct device *dev, pgd_t *pgd, + int pasid, u16 did, int flags) +{ + struct pasid_entry *pte; + + if (!ecap_flts(iommu->ecap)) { + pr_err("No first level translation support on %s\n", + iommu->name); + return -EINVAL; + } + + pte = intel_pasid_get_entry(dev, pasid); + if (WARN_ON(!pte)) + return -EINVAL; + + pasid_clear_entry(pte); + + /* Setup the first level page table pointer: */ + pasid_set_flptr(pte, (u64)__pa(pgd)); + if (flags & PASID_FLAG_SUPERVISOR_MODE) { + if (!ecap_srs(iommu->ecap)) { + pr_err("No supervisor request support on %s\n", + iommu->name); + return -EINVAL; + } + pasid_set_sre(pte); + } + +#ifdef CONFIG_X86 + if (cpu_feature_enabled(X86_FEATURE_LA57)) + pasid_set_flpm(pte, 1); +#endif /* CONFIG_X86 */ + + pasid_set_domain_id(pte, did); + pasid_set_address_width(pte, iommu->agaw); + pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); + + /* Setup Present and PASID Granular Transfer Type: */ + pasid_set_translation_type(pte, 1); + pasid_set_present(pte); + + if (!ecap_coherent(iommu->ecap)) + clflush_cache_range(pte, sizeof(*pte)); + + if (cap_caching_mode(iommu->cap)) { + pasid_cache_invalidation_with_pasid(iommu, did, pasid); + iotlb_invalidation_with_pasid(iommu, did, pasid); + } else { + iommu_flush_write_buffer(iommu); + } + + return 0; +} + /* * Set up the scalable mode pasid entry for second only translation type. */ diff --git a/drivers/iommu/intel-pasid.h b/drivers/iommu/intel-pasid.h index 55bb8715329d..512c63ec8a22 100644 --- a/drivers/iommu/intel-pasid.h +++ b/drivers/iommu/intel-pasid.h @@ -25,6 +25,14 @@ */ #define FLPT_DEFAULT_DID 1 +/* + * The SUPERVISOR_MODE flag indicates a first level translation which + * can be used for access to kernel addresses. It is valid only for + * access to the kernel's static 1:1 mapping of physical memory — not + * to vmalloc or even module mappings. + */ +#define PASID_FLAG_SUPERVISOR_MODE BIT(0) + struct pasid_dir_entry { u64 val; }; @@ -51,6 +59,9 @@ struct pasid_table *intel_pasid_get_table(struct device *dev); int intel_pasid_get_dev_max_id(struct device *dev); struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid); void intel_pasid_clear_entry(struct device *dev, int pasid); +int intel_pasid_setup_first_level(struct intel_iommu *iommu, + struct device *dev, pgd_t *pgd, + int pasid, u16 did, int flags); int intel_pasid_setup_second_level(struct intel_iommu *iommu, struct dmar_domain *domain, struct device *dev, int pasid); diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 5fdd33ed2cce..4ad62396e81e 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -178,6 +178,7 @@ */ #define ecap_smpwc(e) (((e) >> 48) & 0x1) +#define ecap_flts(e) (((e) >> 47) & 0x1) #define ecap_slts(e) (((e) >> 46) & 0x1) #define ecap_smts(e) (((e) >> 43) & 0x1) #define ecap_dit(e) ((e >> 41) & 0x1) -- cgit v1.2.3 From 1c4f88b7f1f9298b56c7dac18c0bcd8d2f75059a Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 10 Dec 2018 09:59:05 +0800 Subject: iommu/vt-d: Shared virtual address in scalable mode This patch enables the current SVA (Shared Virtual Address) implementation to work in the scalable mode. Cc: Ashok Raj Cc: Jacob Pan Cc: Kevin Tian Signed-off-by: Sanjay Kumar Signed-off-by: Liu Yi L Signed-off-by: Lu Baolu Reviewed-by: Ashok Raj Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 38 ----------------------------- drivers/iommu/intel-pasid.c | 2 +- drivers/iommu/intel-pasid.h | 1 - drivers/iommu/intel-svm.c | 58 ++++++++++++++------------------------------- include/linux/intel-iommu.h | 9 +------ 5 files changed, 20 insertions(+), 88 deletions(-) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 21a6853290cc..cec88df671a6 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -5257,18 +5257,6 @@ static void intel_iommu_put_resv_regions(struct device *dev, } #ifdef CONFIG_INTEL_IOMMU_SVM -static inline unsigned long intel_iommu_get_pts(struct device *dev) -{ - int pts, max_pasid; - - max_pasid = intel_pasid_get_dev_max_id(dev); - pts = find_first_bit((unsigned long *)&max_pasid, MAX_NR_PASID_BITS); - if (pts < 5) - return 0; - - return pts - 5; -} - int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sdev) { struct device_domain_info *info; @@ -5300,33 +5288,7 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct intel_svm_dev *sd sdev->sid = PCI_DEVID(info->bus, info->devfn); if (!(ctx_lo & CONTEXT_PASIDE)) { - if (iommu->pasid_state_table) - context[1].hi = (u64)virt_to_phys(iommu->pasid_state_table); - context[1].lo = (u64)virt_to_phys(info->pasid_table->table) | - intel_iommu_get_pts(sdev->dev); - - wmb(); - /* CONTEXT_TT_MULTI_LEVEL and CONTEXT_TT_DEV_IOTLB are both - * extended to permit requests-with-PASID if the PASIDE bit - * is set. which makes sense. For CONTEXT_TT_PASS_THROUGH, - * however, the PASIDE bit is ignored and requests-with-PASID - * are unconditionally blocked. Which makes less sense. - * So convert from CONTEXT_TT_PASS_THROUGH to one of the new - * "guest mode" translation types depending on whether ATS - * is available or not. Annoyingly, we can't use the new - * modes *unless* PASIDE is set. */ - if ((ctx_lo & CONTEXT_TT_MASK) == (CONTEXT_TT_PASS_THROUGH << 2)) { - ctx_lo &= ~CONTEXT_TT_MASK; - if (info->ats_supported) - ctx_lo |= CONTEXT_TT_PT_PASID_DEV_IOTLB << 2; - else - ctx_lo |= CONTEXT_TT_PT_PASID << 2; - } ctx_lo |= CONTEXT_PASIDE; - if (iommu->pasid_state_table) - ctx_lo |= CONTEXT_DINVE; - if (info->pri_supported) - ctx_lo |= CONTEXT_PRS; context[0].lo = ctx_lo; wmb(); iommu->flush.flush_context(iommu, sdev->did, sdev->sid, diff --git a/drivers/iommu/intel-pasid.c b/drivers/iommu/intel-pasid.c index c3dcf4dc2496..53fe5248d8f1 100644 --- a/drivers/iommu/intel-pasid.c +++ b/drivers/iommu/intel-pasid.c @@ -286,7 +286,7 @@ static inline void pasid_clear_entry(struct pasid_entry *pe) WRITE_ONCE(pe->val[7], 0); } -void intel_pasid_clear_entry(struct device *dev, int pasid) +static void intel_pasid_clear_entry(struct device *dev, int pasid) { struct pasid_entry *pe; diff --git a/drivers/iommu/intel-pasid.h b/drivers/iommu/intel-pasid.h index 512c63ec8a22..23537b3f34e3 100644 --- a/drivers/iommu/intel-pasid.h +++ b/drivers/iommu/intel-pasid.h @@ -58,7 +58,6 @@ void intel_pasid_free_table(struct device *dev); struct pasid_table *intel_pasid_get_table(struct device *dev); int intel_pasid_get_dev_max_id(struct device *dev); struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid); -void intel_pasid_clear_entry(struct device *dev, int pasid); int intel_pasid_setup_first_level(struct intel_iommu *iommu, struct device *dev, pgd_t *pgd, int pasid, u16 did, int flags); diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index b7f1d12e24b0..04d6bdb51404 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -29,10 +29,6 @@ #include "intel-pasid.h" -#define PASID_ENTRY_P BIT_ULL(0) -#define PASID_ENTRY_FLPM_5LP BIT_ULL(9) -#define PASID_ENTRY_SRE BIT_ULL(11) - static irqreturn_t prq_event_thread(int irq, void *d); struct pasid_state_entry { @@ -248,20 +244,6 @@ static void intel_invalidate_range(struct mmu_notifier *mn, (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0, 0); } - -static void intel_flush_pasid_dev(struct intel_svm *svm, struct intel_svm_dev *sdev, int pasid) -{ - struct qi_desc desc; - - desc.qw0 = QI_PC_TYPE | QI_PC_DID(sdev->did) | - QI_PC_PASID_SEL | QI_PC_PASID(pasid); - desc.qw1 = 0; - desc.qw2 = 0; - desc.qw3 = 0; - - qi_submit_sync(&desc, svm->iommu); -} - static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) { struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); @@ -281,8 +263,7 @@ static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) */ rcu_read_lock(); list_for_each_entry_rcu(sdev, &svm->devs, list) { - intel_pasid_clear_entry(sdev->dev, svm->pasid); - intel_flush_pasid_dev(svm, sdev, svm->pasid); + intel_pasid_tear_down_entry(svm->iommu, sdev->dev, svm->pasid); intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm); } rcu_read_unlock(); @@ -301,11 +282,9 @@ static LIST_HEAD(global_svm_list); int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ops *ops) { struct intel_iommu *iommu = intel_svm_device_to_iommu(dev); - struct pasid_entry *entry; struct intel_svm_dev *sdev; struct intel_svm *svm = NULL; struct mm_struct *mm = NULL; - u64 pasid_entry_val; int pasid_max; int ret; @@ -414,22 +393,22 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ kfree(sdev); goto out; } - pasid_entry_val = (u64)__pa(mm->pgd) | PASID_ENTRY_P; - } else - pasid_entry_val = (u64)__pa(init_mm.pgd) | - PASID_ENTRY_P | PASID_ENTRY_SRE; - if (cpu_feature_enabled(X86_FEATURE_LA57)) - pasid_entry_val |= PASID_ENTRY_FLPM_5LP; - - entry = intel_pasid_get_entry(dev, svm->pasid); - WRITE_ONCE(entry->val[0], pasid_entry_val); - - /* - * Flush PASID cache when a PASID table entry becomes - * present. - */ - if (cap_caching_mode(iommu->cap)) - intel_flush_pasid_dev(svm, sdev, svm->pasid); + } + + spin_lock(&iommu->lock); + ret = intel_pasid_setup_first_level(iommu, dev, + mm ? mm->pgd : init_mm.pgd, + svm->pasid, FLPT_DEFAULT_DID, + mm ? 0 : PASID_FLAG_SUPERVISOR_MODE); + spin_unlock(&iommu->lock); + if (ret) { + if (mm) + mmu_notifier_unregister(&svm->notifier, mm); + intel_pasid_free_id(svm->pasid); + kfree(svm); + kfree(sdev); + goto out; + } list_add_tail(&svm->list, &global_svm_list); } @@ -475,10 +454,9 @@ int intel_svm_unbind_mm(struct device *dev, int pasid) * to use. We have a *shared* PASID table, because it's * large and has to be physically contiguous. So it's * hard to be as defensive as we might like. */ - intel_flush_pasid_dev(svm, sdev, svm->pasid); + intel_pasid_tear_down_entry(iommu, dev, svm->pasid); intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm); kfree_rcu(sdev, rcu); - intel_pasid_clear_entry(dev, svm->pasid); if (list_empty(&svm->devs)) { intel_pasid_free_id(svm->pasid); diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 4ad62396e81e..cfcf9c1e1872 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -54,14 +54,7 @@ #define CONTEXT_TT_MULTI_LEVEL 0 #define CONTEXT_TT_DEV_IOTLB 1 #define CONTEXT_TT_PASS_THROUGH 2 -/* Extended context entry types */ -#define CONTEXT_TT_PT_PASID 4 -#define CONTEXT_TT_PT_PASID_DEV_IOTLB 5 -#define CONTEXT_TT_MASK (7ULL << 2) - -#define CONTEXT_DINVE (1ULL << 8) -#define CONTEXT_PRS (1ULL << 9) -#define CONTEXT_PASIDE (1ULL << 11) +#define CONTEXT_PASIDE BIT_ULL(3) /* * Intel IOMMU register specification per version 1.0 public spec. -- cgit v1.2.3 From 6d68b88e0993d67e9ebb1240f84240b712fbc8a4 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Mon, 10 Dec 2018 09:59:06 +0800 Subject: iommu/vt-d: Remove deferred invalidation Deferred invalidation is an ECS specific feature. It will not be supported when IOMMU works in scalable mode. As we deprecated the ECS support, remove deferred invalidation and cleanup the code. Cc: Ashok Raj Cc: Jacob Pan Cc: Kevin Tian Cc: Liu Yi L Cc: Sanjay Kumar Signed-off-by: Lu Baolu Reviewed-by: Ashok Raj Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 1 - drivers/iommu/intel-svm.c | 45 --------------------------------------------- include/linux/intel-iommu.h | 8 -------- 3 files changed, 54 deletions(-) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index cec88df671a6..9043e1e9b2be 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1700,7 +1700,6 @@ static void free_dmar_iommu(struct intel_iommu *iommu) if (pasid_supported(iommu)) { if (ecap_prs(iommu->ecap)) intel_svm_finish_prq(iommu); - intel_svm_exit(iommu); } #endif } diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index 04d6bdb51404..5b2e3b2d593b 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -31,15 +31,8 @@ static irqreturn_t prq_event_thread(int irq, void *d); -struct pasid_state_entry { - u64 val; -}; - int intel_svm_init(struct intel_iommu *iommu) { - struct page *pages; - int order; - if (cpu_feature_enabled(X86_FEATURE_GBPAGES) && !cap_fl1gp_support(iommu->cap)) return -EINVAL; @@ -48,39 +41,6 @@ int intel_svm_init(struct intel_iommu *iommu) !cap_5lp_support(iommu->cap)) return -EINVAL; - /* Start at 2 because it's defined as 2^(1+PSS) */ - iommu->pasid_max = 2 << ecap_pss(iommu->ecap); - - /* Eventually I'm promised we will get a multi-level PASID table - * and it won't have to be physically contiguous. Until then, - * limit the size because 8MiB contiguous allocations can be hard - * to come by. The limit of 0x20000, which is 1MiB for each of - * the PASID and PASID-state tables, is somewhat arbitrary. */ - if (iommu->pasid_max > 0x20000) - iommu->pasid_max = 0x20000; - - order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max); - if (ecap_dis(iommu->ecap)) { - pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); - if (pages) - iommu->pasid_state_table = page_address(pages); - else - pr_warn("IOMMU: %s: Failed to allocate PASID state table\n", - iommu->name); - } - - return 0; -} - -int intel_svm_exit(struct intel_iommu *iommu) -{ - int order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max); - - if (iommu->pasid_state_table) { - free_pages((unsigned long)iommu->pasid_state_table, order); - iommu->pasid_state_table = NULL; - } - return 0; } @@ -214,11 +174,6 @@ static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address, { struct intel_svm_dev *sdev; - /* Try deferred invalidate if available */ - if (svm->iommu->pasid_state_table && - !cmpxchg64(&svm->iommu->pasid_state_table[svm->pasid].val, 0, 1ULL << 63)) - return; - rcu_read_lock(); list_for_each_entry_rcu(sdev, &svm->devs, list) intel_flush_svm_range_dev(svm, sdev, address, pages, ih, gl); diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index cfcf9c1e1872..0605f3bf6e79 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -541,15 +541,8 @@ struct intel_iommu { struct iommu_flush flush; #endif #ifdef CONFIG_INTEL_IOMMU_SVM - /* These are large and need to be contiguous, so we allocate just - * one for now. We'll maybe want to rethink that if we truly give - * devices away to userspace processes (e.g. for DPDK) and don't - * want to trust that userspace will use *only* the PASID it was - * told to. But while it's all driver-arbitrated, we're fine. */ - struct pasid_state_entry *pasid_state_table; struct page_req_dsc *prq; unsigned char prq_name[16]; /* Name for PRQ interrupt */ - u32 pasid_max; #endif struct q_inval *qi; /* Queued invalidation info */ u32 *iommu_state; /* Store iommu states between suspend and resume.*/ @@ -663,7 +656,6 @@ void iommu_flush_write_buffer(struct intel_iommu *iommu); #ifdef CONFIG_INTEL_IOMMU_SVM int intel_svm_init(struct intel_iommu *iommu); -int intel_svm_exit(struct intel_iommu *iommu); extern int intel_svm_enable_prq(struct intel_iommu *iommu); extern int intel_svm_finish_prq(struct intel_iommu *iommu); -- cgit v1.2.3 From 69d24c6ed67c1e15f76284fccd62feffe77c8d38 Mon Sep 17 00:00:00 2001 From: Vivek Gautam Date: Thu, 13 Dec 2018 14:35:07 +0530 Subject: dt-bindings: arm-smmu: Add binding doc for Qcom smmu-500 Qcom's implementation of arm,mmu-500 works well with current arm-smmu driver implementation. Adding a soc specific compatible along with arm,mmu-500 makes the bindings future safe. Signed-off-by: Vivek Gautam Reviewed-by: Rob Herring Cc: Will Deacon Acked-by: Will Deacon Signed-off-by: Joerg Roedel --- Documentation/devicetree/bindings/iommu/arm,smmu.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.txt b/Documentation/devicetree/bindings/iommu/arm,smmu.txt index a6504b37cc21..3133f3ba7567 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.txt +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.txt @@ -27,6 +27,10 @@ conditions. "qcom,msm8996-smmu-v2", "qcom,smmu-v2", "qcom,sdm845-smmu-v2", "qcom,smmu-v2". + Qcom SoCs implementing "arm,mmu-500" must also include, + as below, SoC-specific compatibles: + "qcom,sdm845-smmu-500", "arm,mmu-500" + - reg : Base address and size of the SMMU. - #global-interrupts : The number of global interrupts exposed by the -- cgit v1.2.3 From 2713fe37153efb90b7a8427a2f53fa49216faf5c Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Thu, 13 Dec 2018 17:19:48 +0800 Subject: Revert "iommu/io-pgtable-arm: Check for v7s-incapable systems" This reverts commit 82db33dc5e49fb625262d81125625d07a0d6184e. After the commit 29859aeb8a6e ("iommu/io-pgtable-arm-v7s: Abort allocation when table address overflows the PTE"), v7s will return fail if the page table allocation isn't expected. this PHYS_OFFSET check is unnecessary now. And this check may lead to fail. For example, If CONFIG_RANDOMIZE_BASE is enabled, the "memstart_addr" will be updated randomly, then the PHYS_OFFSET may be random. Reported-by: CK Hu Signed-off-by: Yong Wu Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/io-pgtable-arm-v7s.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 445c3bde0480..cec29bf45c9b 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -709,10 +709,6 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, { struct arm_v7s_io_pgtable *data; -#ifdef PHYS_OFFSET - if (upper_32_bits(PHYS_OFFSET)) - return NULL; -#endif if (cfg->ias > ARM_V7S_ADDR_BITS || cfg->oas > ARM_V7S_ADDR_BITS) return NULL; -- cgit v1.2.3 From a6cf933a359f6cf0c642580602a0f5afc40d991a Mon Sep 17 00:00:00 2001 From: Fabrizio Castro Date: Thu, 13 Dec 2018 20:19:28 +0000 Subject: dt-bindings: iommu: ipmmu-vmsa: Add r8a774c0 support Document RZ/G2E (R8A774C0) SoC bindings. Signed-off-by: Fabrizio Castro Reviewed-by: Geert Uytterhoeven Reviewed-by: Simon Horman Signed-off-by: Joerg Roedel --- Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt index e285c8ad1b39..b6bfbec3a849 100644 --- a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt +++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt @@ -15,6 +15,7 @@ Required Properties: - "renesas,ipmmu-r8a7744" for the R8A7744 (RZ/G1N) IPMMU. - "renesas,ipmmu-r8a7745" for the R8A7745 (RZ/G1E) IPMMU. - "renesas,ipmmu-r8a774a1" for the R8A774A1 (RZ/G2M) IPMMU. + - "renesas,ipmmu-r8a774c0" for the R8A774C0 (RZ/G2E) IPMMU. - "renesas,ipmmu-r8a7790" for the R8A7790 (R-Car H2) IPMMU. - "renesas,ipmmu-r8a7791" for the R8A7791 (R-Car M2-W) IPMMU. - "renesas,ipmmu-r8a7793" for the R8A7793 (R-Car M2-N) IPMMU. -- cgit v1.2.3 From b6d39cd82241bfc14a910eab032c53aabe678df7 Mon Sep 17 00:00:00 2001 From: Fabrizio Castro Date: Thu, 13 Dec 2018 20:22:44 +0000 Subject: iommu/ipmmu-vmsa: Hook up r8a774c0 DT matching code Support RZ/G2E (a.k.a. R8A774C0) IPMMU. Signed-off-by: Fabrizio Castro Reviewed-by: Geert Uytterhoeven Reviewed-by: Simon Horman Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 331e86839a8c..f6c419997087 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -753,6 +753,7 @@ static int ipmmu_init_platform_device(struct device *dev, static const struct soc_device_attribute soc_rcar_gen3[] = { { .soc_id = "r8a774a1", }, + { .soc_id = "r8a774c0", }, { .soc_id = "r8a7795", }, { .soc_id = "r8a7796", }, { .soc_id = "r8a77965", }, @@ -763,6 +764,7 @@ static const struct soc_device_attribute soc_rcar_gen3[] = { }; static const struct soc_device_attribute soc_rcar_gen3_whitelist[] = { + { .soc_id = "r8a774c0", }, { .soc_id = "r8a7795", .revision = "ES3.*" }, { .soc_id = "r8a77965", }, { .soc_id = "r8a77990", }, @@ -971,6 +973,9 @@ static const struct of_device_id ipmmu_of_ids[] = { }, { .compatible = "renesas,ipmmu-r8a774a1", .data = &ipmmu_features_rcar_gen3, + }, { + .compatible = "renesas,ipmmu-r8a774c0", + .data = &ipmmu_features_rcar_gen3, }, { .compatible = "renesas,ipmmu-r8a7795", .data = &ipmmu_features_rcar_gen3, -- cgit v1.2.3 From b4ef725eeba158f365da9de1f05149094643ddea Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 28 Nov 2018 13:35:24 +0100 Subject: iommu: Introduce wrappers around dev->iommu_fwspec These wrappers will be used to easily change the location of the field later when all users are converted. Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 14 +++++++------- include/linux/iommu.h | 11 +++++++++++ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index cc25ec6d4c06..304c067a0f85 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1976,7 +1976,7 @@ const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode) int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, const struct iommu_ops *ops) { - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); if (fwspec) return ops == fwspec->ops ? 0 : -EINVAL; @@ -1988,26 +1988,26 @@ int iommu_fwspec_init(struct device *dev, struct fwnode_handle *iommu_fwnode, of_node_get(to_of_node(iommu_fwnode)); fwspec->iommu_fwnode = iommu_fwnode; fwspec->ops = ops; - dev->iommu_fwspec = fwspec; + dev_iommu_fwspec_set(dev, fwspec); return 0; } EXPORT_SYMBOL_GPL(iommu_fwspec_init); void iommu_fwspec_free(struct device *dev) { - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); if (fwspec) { fwnode_handle_put(fwspec->iommu_fwnode); kfree(fwspec); - dev->iommu_fwspec = NULL; + dev_iommu_fwspec_set(dev, NULL); } } EXPORT_SYMBOL_GPL(iommu_fwspec_free); int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids) { - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); size_t size; int i; @@ -2016,11 +2016,11 @@ int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids) size = offsetof(struct iommu_fwspec, ids[fwspec->num_ids + num_ids]); if (size > sizeof(*fwspec)) { - fwspec = krealloc(dev->iommu_fwspec, size, GFP_KERNEL); + fwspec = krealloc(fwspec, size, GFP_KERNEL); if (!fwspec) return -ENOMEM; - dev->iommu_fwspec = fwspec; + dev_iommu_fwspec_set(dev, fwspec); } for (i = 0; i < num_ids; i++) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 11db18b9ffe8..26225f762cd7 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -398,6 +398,17 @@ void iommu_fwspec_free(struct device *dev); int iommu_fwspec_add_ids(struct device *dev, u32 *ids, int num_ids); const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode); +static inline struct iommu_fwspec *dev_iommu_fwspec_get(struct device *dev) +{ + return dev->iommu_fwspec; +} + +static inline void dev_iommu_fwspec_set(struct device *dev, + struct iommu_fwspec *fwspec) +{ + dev->iommu_fwspec = fwspec; +} + #else /* CONFIG_IOMMU_API */ struct iommu_ops {}; -- cgit v1.2.3 From 8097e53eaba2bbae99dee661778ca20490b73951 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 29 Nov 2018 14:01:00 +0100 Subject: ACPI/IORT: Use helper functions to access dev->iommu_fwspec Use the new helpers dev_iommu_fwspec_get()/set() to access the dev->iommu_fwspec pointer. This makes it easier to move that pointer later into another struct. Acked-by: Hanjun Guo Signed-off-by: Joerg Roedel --- drivers/acpi/arm64/iort.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 2a361e22d38d..8fbb9d0430cf 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -779,7 +779,7 @@ static inline bool iort_iommu_driver_enabled(u8 type) static struct acpi_iort_node *iort_get_msi_resv_iommu(struct device *dev) { struct acpi_iort_node *iommu; - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); iommu = iort_get_iort_node(fwspec->iommu_fwnode); @@ -794,9 +794,10 @@ static struct acpi_iort_node *iort_get_msi_resv_iommu(struct device *dev) return NULL; } -static inline const struct iommu_ops *iort_fwspec_iommu_ops( - struct iommu_fwspec *fwspec) +static inline const struct iommu_ops *iort_fwspec_iommu_ops(struct device *dev) { + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + return (fwspec && fwspec->ops) ? fwspec->ops : NULL; } @@ -824,6 +825,7 @@ static inline int iort_add_device_replay(const struct iommu_ops *ops, */ int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) { + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct acpi_iort_its_group *its; struct acpi_iort_node *iommu_node, *its_node = NULL; int i, resv = 0; @@ -841,9 +843,9 @@ int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) * a given PCI or named component may map IDs to. */ - for (i = 0; i < dev->iommu_fwspec->num_ids; i++) { + for (i = 0; i < fwspec->num_ids; i++) { its_node = iort_node_map_id(iommu_node, - dev->iommu_fwspec->ids[i], + fwspec->ids[i], NULL, IORT_MSI_TYPE); if (its_node) break; @@ -874,8 +876,7 @@ int iort_iommu_msi_get_resv_regions(struct device *dev, struct list_head *head) return (resv == its->its_count) ? resv : -ENODEV; } #else -static inline const struct iommu_ops *iort_fwspec_iommu_ops( - struct iommu_fwspec *fwspec) +static inline const struct iommu_ops *iort_fwspec_iommu_ops(struct device *dev); { return NULL; } static inline int iort_add_device_replay(const struct iommu_ops *ops, struct device *dev) @@ -1045,7 +1046,7 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev) * If we already translated the fwspec there * is nothing left to do, return the iommu_ops. */ - ops = iort_fwspec_iommu_ops(dev->iommu_fwspec); + ops = iort_fwspec_iommu_ops(dev); if (ops) return ops; @@ -1084,7 +1085,7 @@ const struct iommu_ops *iort_iommu_configure(struct device *dev) * add_device callback for dev, replay it to get things in order. */ if (!err) { - ops = iort_fwspec_iommu_ops(dev->iommu_fwspec); + ops = iort_fwspec_iommu_ops(dev); err = iort_add_device_replay(ops, dev); } -- cgit v1.2.3 From 9b468f7d9cf1f089b7287865776eb100504681b7 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 29 Nov 2018 14:01:00 +0100 Subject: iommu/arm-smmu: Use helper functions to access dev->iommu_fwspec Use the new helpers dev_iommu_fwspec_get()/set() to access the dev->iommu_fwspec pointer. This makes it easier to move that pointer later into another struct. Cc: Will Deacon Cc: Robin Murphy Acked-by: Will Deacon Signed-off-by: Joerg Roedel --- drivers/iommu/arm-smmu-v3.c | 16 +++++++++------- drivers/iommu/arm-smmu.c | 12 ++++++------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 1189c06079d4..ca147978120e 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1696,24 +1696,26 @@ static void arm_smmu_install_ste_for_dev(struct iommu_fwspec *fwspec) static void arm_smmu_detach_dev(struct device *dev) { - struct arm_smmu_master_data *master = dev->iommu_fwspec->iommu_priv; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct arm_smmu_master_data *master = fwspec->iommu_priv; master->ste.assigned = false; - arm_smmu_install_ste_for_dev(dev->iommu_fwspec); + arm_smmu_install_ste_for_dev(fwspec); } static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret = 0; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_device *smmu; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_master_data *master; struct arm_smmu_strtab_ent *ste; - if (!dev->iommu_fwspec) + if (!fwspec) return -ENOENT; - master = dev->iommu_fwspec->iommu_priv; + master = fwspec->iommu_priv; smmu = master->smmu; ste = &master->ste; @@ -1753,7 +1755,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) ste->s2_cfg = &smmu_domain->s2_cfg; } - arm_smmu_install_ste_for_dev(dev->iommu_fwspec); + arm_smmu_install_ste_for_dev(fwspec); out_unlock: mutex_unlock(&smmu_domain->init_mutex); return ret; @@ -1844,7 +1846,7 @@ static int arm_smmu_add_device(struct device *dev) int i, ret; struct arm_smmu_device *smmu; struct arm_smmu_master_data *master; - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct iommu_group *group; if (!fwspec || fwspec->ops != &arm_smmu_ops) @@ -1895,7 +1897,7 @@ static int arm_smmu_add_device(struct device *dev) static void arm_smmu_remove_device(struct device *dev) { - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_master_data *master; struct arm_smmu_device *smmu; diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 4a2e143fdf52..44bff7de5fe2 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1108,7 +1108,7 @@ static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) static int arm_smmu_master_alloc_smes(struct device *dev) { - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; struct arm_smmu_device *smmu = cfg->smmu; struct arm_smmu_smr *smrs = smmu->smrs; @@ -1211,7 +1211,7 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret; - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_device *smmu; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); @@ -1385,7 +1385,7 @@ static int arm_smmu_add_device(struct device *dev) { struct arm_smmu_device *smmu; struct arm_smmu_master_cfg *cfg; - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); int i, ret; if (using_legacy_binding) { @@ -1396,7 +1396,7 @@ static int arm_smmu_add_device(struct device *dev) * will allocate/initialise a new one. Thus we need to update fwspec for * later use. */ - fwspec = dev->iommu_fwspec; + fwspec = dev_iommu_fwspec_get(dev); if (ret) goto out_free; } else if (fwspec && fwspec->ops == &arm_smmu_ops) { @@ -1450,7 +1450,7 @@ out_free: static void arm_smmu_remove_device(struct device *dev) { - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_master_cfg *cfg; struct arm_smmu_device *smmu; @@ -1470,7 +1470,7 @@ static void arm_smmu_remove_device(struct device *dev) static struct iommu_group *arm_smmu_device_group(struct device *dev) { - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct arm_smmu_device *smmu = fwspec_smmu(fwspec); struct iommu_group *group = NULL; int i, idx; -- cgit v1.2.3 From 98cc4f7196422ee20e00e2067e484e3136dbe4a5 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 29 Nov 2018 14:01:00 +0100 Subject: iommu/dma: Use helper functions to access dev->iommu_fwspec Use the new helpers dev_iommu_fwspec_get()/set() to access the dev->iommu_fwspec pointer. This makes it easier to move that pointer later into another struct. Signed-off-by: Joerg Roedel --- drivers/iommu/dma-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 4afb1a8a8da8..13787d3ee338 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -177,7 +177,7 @@ EXPORT_SYMBOL(iommu_put_dma_cookie); void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list) { - if (!is_of_node(dev->iommu_fwspec->iommu_fwnode)) + if (!is_of_node(dev_iommu_fwspec_get(dev)->iommu_fwnode)) iort_iommu_msi_get_resv_regions(dev, list); } -- cgit v1.2.3 From df90365580623b5c53cd9a46c14375191123d776 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 29 Nov 2018 14:01:00 +0100 Subject: iommu/ipmmu-vmsa: Use helper functions to access dev->iommu_fwspec Use the new helpers dev_iommu_fwspec_get()/set() to access the dev->iommu_fwspec pointer. This makes it easier to move that pointer later into another struct. Signed-off-by: Joerg Roedel --- drivers/iommu/ipmmu-vmsa.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index c67c3aa8537a..4b264b63035b 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -82,7 +82,9 @@ static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom) static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev) { - return dev->iommu_fwspec ? dev->iommu_fwspec->iommu_priv : NULL; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + + return fwspec ? fwspec->iommu_priv : NULL; } #define TLB_LOOP_TIMEOUT 100 /* 100us */ @@ -641,7 +643,7 @@ static void ipmmu_domain_free(struct iommu_domain *io_domain) static int ipmmu_attach_device(struct iommu_domain *io_domain, struct device *dev) { - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct ipmmu_vmsa_device *mmu = to_ipmmu(dev); struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); unsigned int i; @@ -690,7 +692,7 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain, static void ipmmu_detach_device(struct iommu_domain *io_domain, struct device *dev) { - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain); unsigned int i; @@ -742,13 +744,15 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, static int ipmmu_init_platform_device(struct device *dev, struct of_phandle_args *args) { + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct platform_device *ipmmu_pdev; ipmmu_pdev = of_find_device_by_node(args->np); if (!ipmmu_pdev) return -ENODEV; - dev->iommu_fwspec->iommu_priv = platform_get_drvdata(ipmmu_pdev); + fwspec->iommu_priv = platform_get_drvdata(ipmmu_pdev); + return 0; } -- cgit v1.2.3 From a9bf2eec5a6fc01a0a5250eaf0bf61dfd382a78a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 29 Nov 2018 14:01:00 +0100 Subject: iommu/mediatek: Use helper functions to access dev->iommu_fwspec Use the new helpers dev_iommu_fwspec_get()/set() to access the dev->iommu_fwspec pointer. This makes it easier to move that pointer later into another struct. Cc: Matthias Brugger Tested-by: Yong Wu Signed-off-by: Joerg Roedel --- drivers/iommu/mtk_iommu.c | 21 ++++++++++++--------- drivers/iommu/mtk_iommu_v1.c | 28 ++++++++++++++++------------ 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 44bd5b9166bb..0783fba05d19 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -244,7 +244,7 @@ static void mtk_iommu_config(struct mtk_iommu_data *data, { struct mtk_smi_larb_iommu *larb_mmu; unsigned int larbid, portid; - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); int i; for (i = 0; i < fwspec->num_ids; ++i) { @@ -336,7 +336,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { struct mtk_iommu_domain *dom = to_mtk_domain(domain); - struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv; + struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv; if (!data) return -ENODEV; @@ -355,7 +355,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, static void mtk_iommu_detach_device(struct iommu_domain *domain, struct device *dev) { - struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv; + struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv; if (!data) return; @@ -417,13 +417,14 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, static int mtk_iommu_add_device(struct device *dev) { + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct mtk_iommu_data *data; struct iommu_group *group; - if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops) + if (!fwspec || fwspec->ops != &mtk_iommu_ops) return -ENODEV; /* Not a iommu client device */ - data = dev->iommu_fwspec->iommu_priv; + data = fwspec->iommu_priv; iommu_device_link(&data->iommu, dev); group = iommu_group_get_for_dev(dev); @@ -436,12 +437,13 @@ static int mtk_iommu_add_device(struct device *dev) static void mtk_iommu_remove_device(struct device *dev) { + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct mtk_iommu_data *data; - if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops) + if (!fwspec || fwspec->ops != &mtk_iommu_ops) return; - data = dev->iommu_fwspec->iommu_priv; + data = fwspec->iommu_priv; iommu_device_unlink(&data->iommu, dev); iommu_group_remove_device(dev); @@ -468,6 +470,7 @@ static struct iommu_group *mtk_iommu_device_group(struct device *dev) static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) { + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct platform_device *m4updev; if (args->args_count != 1) { @@ -476,13 +479,13 @@ static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) return -EINVAL; } - if (!dev->iommu_fwspec->iommu_priv) { + if (!fwspec->iommu_priv) { /* Get the m4u device */ m4updev = of_find_device_by_node(args->np); if (WARN_ON(!m4updev)) return -EINVAL; - dev->iommu_fwspec->iommu_priv = platform_get_drvdata(m4updev); + fwspec->iommu_priv = platform_get_drvdata(m4updev); } return iommu_fwspec_add_ids(dev, args->args, 1); diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 6d4551e8ef85..83d8a4cb2732 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -208,7 +208,7 @@ static void mtk_iommu_config(struct mtk_iommu_data *data, { struct mtk_smi_larb_iommu *larb_mmu; unsigned int larbid, portid; - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); int i; for (i = 0; i < fwspec->num_ids; ++i) { @@ -273,7 +273,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { struct mtk_iommu_domain *dom = to_mtk_domain(domain); - struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv; + struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv; int ret; if (!data) @@ -295,7 +295,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, static void mtk_iommu_detach_device(struct iommu_domain *domain, struct device *dev) { - struct mtk_iommu_data *data = dev->iommu_fwspec->iommu_priv; + struct mtk_iommu_data *data = dev_iommu_fwspec_get(dev)->iommu_priv; if (!data) return; @@ -373,6 +373,7 @@ static struct iommu_ops mtk_iommu_ops; static int mtk_iommu_create_mapping(struct device *dev, struct of_phandle_args *args) { + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct mtk_iommu_data *data; struct platform_device *m4updev; struct dma_iommu_mapping *mtk_mapping; @@ -385,28 +386,29 @@ static int mtk_iommu_create_mapping(struct device *dev, return -EINVAL; } - if (!dev->iommu_fwspec) { + if (!fwspec) { ret = iommu_fwspec_init(dev, &args->np->fwnode, &mtk_iommu_ops); if (ret) return ret; - } else if (dev->iommu_fwspec->ops != &mtk_iommu_ops) { + fwspec = dev_iommu_fwspec_get(dev); + } else if (dev_iommu_fwspec_get(dev)->ops != &mtk_iommu_ops) { return -EINVAL; } - if (!dev->iommu_fwspec->iommu_priv) { + if (!fwspec->iommu_priv) { /* Get the m4u device */ m4updev = of_find_device_by_node(args->np); if (WARN_ON(!m4updev)) return -EINVAL; - dev->iommu_fwspec->iommu_priv = platform_get_drvdata(m4updev); + fwspec->iommu_priv = platform_get_drvdata(m4updev); } ret = iommu_fwspec_add_ids(dev, args->args, 1); if (ret) return ret; - data = dev->iommu_fwspec->iommu_priv; + data = fwspec->iommu_priv; m4udev = data->dev; mtk_mapping = m4udev->archdata.iommu; if (!mtk_mapping) { @@ -424,6 +426,7 @@ static int mtk_iommu_create_mapping(struct device *dev, static int mtk_iommu_add_device(struct device *dev) { + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct dma_iommu_mapping *mtk_mapping; struct of_phandle_args iommu_spec; struct of_phandle_iterator it; @@ -442,7 +445,7 @@ static int mtk_iommu_add_device(struct device *dev) of_node_put(iommu_spec.np); } - if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops) + if (!fwspec || fwspec->ops != &mtk_iommu_ops) return -ENODEV; /* Not a iommu client device */ /* @@ -460,7 +463,7 @@ static int mtk_iommu_add_device(struct device *dev) if (err) return err; - data = dev->iommu_fwspec->iommu_priv; + data = fwspec->iommu_priv; mtk_mapping = data->dev->archdata.iommu; err = arm_iommu_attach_device(dev, mtk_mapping); if (err) { @@ -473,12 +476,13 @@ static int mtk_iommu_add_device(struct device *dev) static void mtk_iommu_remove_device(struct device *dev) { + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct mtk_iommu_data *data; - if (!dev->iommu_fwspec || dev->iommu_fwspec->ops != &mtk_iommu_ops) + if (!fwspec || fwspec->ops != &mtk_iommu_ops) return; - data = dev->iommu_fwspec->iommu_priv; + data = fwspec->iommu_priv; iommu_device_unlink(&data->iommu, dev); iommu_group_remove_device(dev); -- cgit v1.2.3 From 5c7e6bd71bfd96a65adb62593ea1b7d51a445b26 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 29 Nov 2018 14:01:00 +0100 Subject: iommu/of: Use helper functions to access dev->iommu_fwspec Use the new helpers dev_iommu_fwspec_get()/set() to access the dev->iommu_fwspec pointer. This makes it easier to move that pointer later into another struct. Signed-off-by: Joerg Roedel --- drivers/iommu/of_iommu.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index c5dd63072529..8b071f3dcf9e 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -164,7 +164,7 @@ const struct iommu_ops *of_iommu_configure(struct device *dev, struct device_node *master_np) { const struct iommu_ops *ops = NULL; - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); int err = NO_IOMMU; if (!master_np) @@ -208,14 +208,18 @@ const struct iommu_ops *of_iommu_configure(struct device *dev, } } + /* * Two success conditions can be represented by non-negative err here: * >0 : there is no IOMMU, or one was unavailable for non-fatal reasons * 0 : we found an IOMMU, and dev->fwspec is initialised appropriately * <0 : any actual error */ - if (!err) - ops = dev->iommu_fwspec->ops; + if (!err) { + /* The fwspec pointer changed, read it again */ + fwspec = dev_iommu_fwspec_get(dev); + ops = fwspec->ops; + } /* * If we have reason to believe the IOMMU driver missed the initial * add_device callback for dev, replay it to get things in order. -- cgit v1.2.3 From 2000e5f70322dcd612a9ccddfb56bf6716e0da9a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 29 Nov 2018 14:01:00 +0100 Subject: iommu/qcom: Use helper functions to access dev->iommu_fwspec Use the new helpers dev_iommu_fwspec_get()/set() to access the dev->iommu_fwspec pointer. This makes it easier to move that pointer later into another struct. Cc: Rob Clark Signed-off-by: Joerg Roedel --- drivers/iommu/qcom_iommu.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c index 4c8f4fc54106..d8595f0a987d 100644 --- a/drivers/iommu/qcom_iommu.c +++ b/drivers/iommu/qcom_iommu.c @@ -354,7 +354,8 @@ static void qcom_iommu_domain_free(struct iommu_domain *domain) static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) { - struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec); + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec); struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain); int ret; @@ -365,7 +366,7 @@ static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev /* Ensure that the domain is finalized */ pm_runtime_get_sync(qcom_iommu->dev); - ret = qcom_iommu_init_domain(domain, qcom_iommu, dev->iommu_fwspec); + ret = qcom_iommu_init_domain(domain, qcom_iommu, fwspec); pm_runtime_put_sync(qcom_iommu->dev); if (ret < 0) return ret; @@ -387,7 +388,7 @@ static int qcom_iommu_attach_dev(struct iommu_domain *domain, struct device *dev static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *dev) { - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct qcom_iommu_dev *qcom_iommu = to_iommu(fwspec); struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain); unsigned i; @@ -500,7 +501,7 @@ static bool qcom_iommu_capable(enum iommu_cap cap) static int qcom_iommu_add_device(struct device *dev) { - struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec); + struct qcom_iommu_dev *qcom_iommu = to_iommu(dev_iommu_fwspec_get(dev)); struct iommu_group *group; struct device_link *link; @@ -531,7 +532,7 @@ static int qcom_iommu_add_device(struct device *dev) static void qcom_iommu_remove_device(struct device *dev) { - struct qcom_iommu_dev *qcom_iommu = to_iommu(dev->iommu_fwspec); + struct qcom_iommu_dev *qcom_iommu = to_iommu(dev_iommu_fwspec_get(dev)); if (!qcom_iommu) return; @@ -543,6 +544,7 @@ static void qcom_iommu_remove_device(struct device *dev) static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) { + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct qcom_iommu_dev *qcom_iommu; struct platform_device *iommu_pdev; unsigned asid = args->args[0]; @@ -568,14 +570,14 @@ static int qcom_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) WARN_ON(asid > qcom_iommu->num_ctxs)) return -EINVAL; - if (!dev->iommu_fwspec->iommu_priv) { - dev->iommu_fwspec->iommu_priv = qcom_iommu; + if (!fwspec->iommu_priv) { + fwspec->iommu_priv = qcom_iommu; } else { /* make sure devices iommus dt node isn't referring to * multiple different iommu devices. Multiple context * banks are ok, but multiple devices are not: */ - if (WARN_ON(qcom_iommu != dev->iommu_fwspec->iommu_priv)) + if (WARN_ON(qcom_iommu != fwspec->iommu_priv)) return -EINVAL; } -- cgit v1.2.3 From db5d6a70043a0073e59af99cd4fd6bcb5bd3ace2 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 29 Nov 2018 14:01:00 +0100 Subject: iommu/tegra: Use helper functions to access dev->iommu_fwspec Use the new helpers dev_iommu_fwspec_get()/set() to access the dev->iommu_fwspec pointer. This makes it easier to move that pointer later into another struct. Cc: Thierry Reding Acked-by: Thierry Reding Signed-off-by: Joerg Roedel --- drivers/iommu/tegra-smmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 0d03341317c4..0d026cb2dfff 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -846,7 +846,7 @@ static struct iommu_group *tegra_smmu_group_get(struct tegra_smmu *smmu, static struct iommu_group *tegra_smmu_device_group(struct device *dev) { - struct iommu_fwspec *fwspec = dev->iommu_fwspec; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); struct tegra_smmu *smmu = dev->archdata.iommu; struct iommu_group *group; -- cgit v1.2.3 From dbba197edf32209d110727a02d3a91de4c88520f Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Nov 2018 12:51:52 +0100 Subject: driver core: Introduce device_iommu_mapped() function Some places in the kernel check the iommu_group pointer in 'struct device' in order to find out whether a device is mapped by an IOMMU. This is not good way to make this check, as the pointer will be moved to 'struct dev_iommu_data'. This way to make the check is also not very readable. Introduce an explicit function to perform this check. Acked-by: Greg Kroah-Hartman Acked-by: Robin Murphy Signed-off-by: Joerg Roedel --- include/linux/device.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/linux/device.h b/include/linux/device.h index 1b25c7a43f4c..6cb4640b6160 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -1058,6 +1058,16 @@ static inline struct device *kobj_to_dev(struct kobject *kobj) return container_of(kobj, struct device, kobj); } +/** + * device_iommu_mapped - Returns true when the device DMA is translated + * by an IOMMU + * @dev: Device to perform the check on + */ +static inline bool device_iommu_mapped(struct device *dev) +{ + return (dev->iommu_group != NULL); +} + /* Get the wakeup routines, which depend on struct device */ #include -- cgit v1.2.3 From 170ecbd5ff09aa9c8c1025a8563f76e01a24bb50 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Nov 2018 14:29:01 +0100 Subject: iommu/of: Use device_iommu_mapped() Use Use device_iommu_mapped() to check if the device is already mapped by an IOMMU. Acked-by: Robin Murphy Signed-off-by: Joerg Roedel --- drivers/iommu/of_iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 8b071f3dcf9e..dbe73456f1a1 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -224,7 +224,7 @@ const struct iommu_ops *of_iommu_configure(struct device *dev, * If we have reason to believe the IOMMU driver missed the initial * add_device callback for dev, replay it to get things in order. */ - if (ops && ops->add_device && dev->bus && !dev->iommu_group) + if (ops && ops->add_device && dev->bus && !device_iommu_mapped(dev)) err = ops->add_device(dev); /* Ignore all other errors apart from EPROBE_DEFER */ -- cgit v1.2.3 From 13e6a84d697e8eb8b61a3fe55b4109a666c4851d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Nov 2018 13:14:05 +0100 Subject: ACPI/IORT: Use device_iommu_mapped() Replace the iommu-check with a proper and readable function call. Cc: Lorenzo Pieralisi Acked-by: Robin Murphy Acked-by: Hanjun Guo Signed-off-by: Joerg Roedel --- drivers/acpi/arm64/iort.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index 8fbb9d0430cf..c057d3649a2e 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -806,7 +806,7 @@ static inline int iort_add_device_replay(const struct iommu_ops *ops, { int err = 0; - if (ops->add_device && dev->bus && !dev->iommu_group) + if (ops->add_device && dev->bus && !device_iommu_mapped(dev)) err = ops->add_device(dev); return err; -- cgit v1.2.3 From bf8763d8f8376e98ea2a8e0fc4803f25ff91393e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Nov 2018 14:23:19 +0100 Subject: powerpc/iommu: Use device_iommu_mapped() Use the new function to replace the open-coded iommu check. Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Russell Currey Cc: Sam Bobroff Acked-by: Robin Murphy Signed-off-by: Joerg Roedel --- arch/powerpc/kernel/eeh.c | 2 +- arch/powerpc/kernel/iommu.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index 6cae6b56ffd6..23fe62f11486 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -1472,7 +1472,7 @@ static int dev_has_iommu_table(struct device *dev, void *data) if (!dev) return 0; - if (dev->iommu_group) { + if (device_iommu_mapped(dev)) { *ppdev = pdev; return 1; } diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index f0dc680e659a..48d58d1dcac2 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -1086,7 +1086,7 @@ int iommu_add_device(struct device *dev) if (!device_is_registered(dev)) return -ENOENT; - if (dev->iommu_group) { + if (device_iommu_mapped(dev)) { pr_debug("%s: Skipping device %s with iommu group %d\n", __func__, dev_name(dev), iommu_group_id(dev->iommu_group)); @@ -1129,7 +1129,7 @@ void iommu_del_device(struct device *dev) * and we needn't detach them from the associated * IOMMU groups */ - if (!dev->iommu_group) { + if (!device_iommu_mapped(dev)) { pr_debug("iommu_tce: skipping device %s with no tbl\n", dev_name(dev)); return; @@ -1148,7 +1148,7 @@ static int tce_iommu_bus_notifier(struct notifier_block *nb, case BUS_NOTIFY_ADD_DEVICE: return iommu_add_device(dev); case BUS_NOTIFY_DEL_DEVICE: - if (dev->iommu_group) + if (device_iommu_mapped(dev)) iommu_del_device(dev); return 0; default: -- cgit v1.2.3 From 05afde1a7ef3ddde5e7657b9faeb6347423a9acb Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Nov 2018 13:16:38 +0100 Subject: xhci: Use device_iommu_mapped() Replace the dev->iommu_group check with a proper function call that better reprensents its purpose. Cc: Mathias Nyman Acked-by: Robin Murphy Acked-by: Mathias Nyman Signed-off-by: Joerg Roedel --- drivers/usb/host/xhci.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 0420eefa647a..6216e8254128 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -244,7 +244,7 @@ static void xhci_zero_64b_regs(struct xhci_hcd *xhci) * an iommu. Doing anything when there is no iommu is definitely * unsafe... */ - if (!(xhci->quirks & XHCI_ZERO_64B_REGS) || !dev->iommu_group) + if (!(xhci->quirks & XHCI_ZERO_64B_REGS) || !device_iommu_mapped(dev)) return; xhci_info(xhci, "Zeroing 64bit base registers, expecting fault\n"); -- cgit v1.2.3 From f884f6ee62604aec60fe1760f94724be192d97c0 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Nov 2018 14:29:01 +0100 Subject: dmaengine: sh: rcar-dmac: Use device_iommu_mapped() Use Use device_iommu_mapped() to check if the device is already mapped by an IOMMU. Acked-by: Vinod Koul Signed-off-by: Joerg Roedel --- drivers/dma/sh/rcar-dmac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index 74fa2b1a6a86..2b4f25698169 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -1809,7 +1809,7 @@ static int rcar_dmac_probe(struct platform_device *pdev) * level we can't disable it selectively, so ignore channel 0 for now if * the device is part of an IOMMU group. */ - if (pdev->dev.iommu_group) { + if (device_iommu_mapped(&pdev->dev)) { dmac->n_channels--; channels_offset = 1; } -- cgit v1.2.3 From 6954cf9bfda153f9544c63761aabf0199710aec3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 5 Dec 2018 13:57:43 +0100 Subject: iommu/sysfs: Rename iommu_release_device() Remove the iommu_ prefix from the function and a few other static data structures so that the iommu_release_device name can be re-used in iommu core code. Signed-off-by: Joerg Roedel --- drivers/iommu/iommu-sysfs.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/iommu-sysfs.c b/drivers/iommu/iommu-sysfs.c index c298330ba2b7..44127d54e943 100644 --- a/drivers/iommu/iommu-sysfs.c +++ b/drivers/iommu/iommu-sysfs.c @@ -22,25 +22,25 @@ static struct attribute *devices_attr[] = { NULL, }; -static const struct attribute_group iommu_devices_attr_group = { +static const struct attribute_group devices_attr_group = { .name = "devices", .attrs = devices_attr, }; -static const struct attribute_group *iommu_dev_groups[] = { - &iommu_devices_attr_group, +static const struct attribute_group *dev_groups[] = { + &devices_attr_group, NULL, }; -static void iommu_release_device(struct device *dev) +static void release_device(struct device *dev) { kfree(dev); } static struct class iommu_class = { .name = "iommu", - .dev_release = iommu_release_device, - .dev_groups = iommu_dev_groups, + .dev_release = release_device, + .dev_groups = dev_groups, }; static int __init iommu_dev_init(void) -- cgit v1.2.3 From cc5aed44a3a8e4fca721636cf881a52f8d68a098 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 Nov 2018 10:31:59 +0100 Subject: iommu: Consolitate ->add/remove_device() calls Put them into separate functions and call those where the plain ops have been called before. Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 51 +++++++++++++++++++++++++-------------------------- include/linux/iommu.h | 3 +++ 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 304c067a0f85..a2131751dcff 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -111,6 +111,23 @@ void iommu_device_unregister(struct iommu_device *iommu) spin_unlock(&iommu_device_lock); } +int iommu_probe_device(struct device *dev) +{ + const struct iommu_ops *ops = dev->bus->iommu_ops; + + WARN_ON(dev->iommu_group); + + return ops->add_device(dev); +} + +void iommu_release_device(struct device *dev) +{ + const struct iommu_ops *ops = dev->bus->iommu_ops; + + if (dev->iommu_group) + ops->remove_device(dev); +} + static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, unsigned type); static int __iommu_attach_device(struct iommu_domain *domain, @@ -1118,16 +1135,7 @@ struct iommu_domain *iommu_group_default_domain(struct iommu_group *group) static int add_iommu_group(struct device *dev, void *data) { - struct iommu_callback_data *cb = data; - const struct iommu_ops *ops = cb->ops; - int ret; - - if (!ops->add_device) - return 0; - - WARN_ON(dev->iommu_group); - - ret = ops->add_device(dev); + int ret = iommu_probe_device(dev); /* * We ignore -ENODEV errors for now, as they just mean that the @@ -1142,11 +1150,7 @@ static int add_iommu_group(struct device *dev, void *data) static int remove_iommu_group(struct device *dev, void *data) { - struct iommu_callback_data *cb = data; - const struct iommu_ops *ops = cb->ops; - - if (ops->remove_device && dev->iommu_group) - ops->remove_device(dev); + iommu_release_device(dev); return 0; } @@ -1154,27 +1158,22 @@ static int remove_iommu_group(struct device *dev, void *data) static int iommu_bus_notifier(struct notifier_block *nb, unsigned long action, void *data) { + unsigned long group_action = 0; struct device *dev = data; - const struct iommu_ops *ops = dev->bus->iommu_ops; struct iommu_group *group; - unsigned long group_action = 0; /* * ADD/DEL call into iommu driver ops if provided, which may * result in ADD/DEL notifiers to group->notifier */ if (action == BUS_NOTIFY_ADD_DEVICE) { - if (ops->add_device) { - int ret; + int ret; - ret = ops->add_device(dev); - return (ret) ? NOTIFY_DONE : NOTIFY_OK; - } + ret = iommu_probe_device(dev); + return (ret) ? NOTIFY_DONE : NOTIFY_OK; } else if (action == BUS_NOTIFY_REMOVED_DEVICE) { - if (ops->remove_device && dev->iommu_group) { - ops->remove_device(dev); - return 0; - } + iommu_release_device(dev); + return NOTIFY_OK; } /* diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 26225f762cd7..e90da6b6f3d1 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -409,6 +409,9 @@ static inline void dev_iommu_fwspec_set(struct device *dev, dev->iommu_fwspec = fwspec; } +int iommu_probe_device(struct device *dev); +void iommu_release_device(struct device *dev); + #else /* CONFIG_IOMMU_API */ struct iommu_ops {}; -- cgit v1.2.3 From 641fb0efbff063ed57f108c2eb4a4d26dbd5badd Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 5 Dec 2018 14:39:45 +0100 Subject: iommu/of: Don't call iommu_ops->add_device directly Make sure to invoke this call-back through the proper function of the IOMMU-API. Signed-off-by: Joerg Roedel --- drivers/iommu/of_iommu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index dbe73456f1a1..d8947b28db2d 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -222,10 +222,10 @@ const struct iommu_ops *of_iommu_configure(struct device *dev, } /* * If we have reason to believe the IOMMU driver missed the initial - * add_device callback for dev, replay it to get things in order. + * probe for dev, replay it to get things in order. */ - if (ops && ops->add_device && dev->bus && !device_iommu_mapped(dev)) - err = ops->add_device(dev); + if (dev->bus && !device_iommu_mapped(dev)) + err = iommu_probe_device(dev); /* Ignore all other errors apart from EPROBE_DEFER */ if (err == -EPROBE_DEFER) { -- cgit v1.2.3 From d2e1a003af569e912d5f115a3c20b89a19f5caa0 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 5 Dec 2018 14:39:45 +0100 Subject: ACPI/IORT: Don't call iommu_ops->add_device directly Make sure to invoke this call-back through the proper function of the IOMMU-API. Acked-by: Hanjun Guo Signed-off-by: Joerg Roedel --- drivers/acpi/arm64/iort.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/acpi/arm64/iort.c b/drivers/acpi/arm64/iort.c index c057d3649a2e..ec248c067f6d 100644 --- a/drivers/acpi/arm64/iort.c +++ b/drivers/acpi/arm64/iort.c @@ -806,8 +806,8 @@ static inline int iort_add_device_replay(const struct iommu_ops *ops, { int err = 0; - if (ops->add_device && dev->bus && !device_iommu_mapped(dev)) - err = ops->add_device(dev); + if (dev->bus && !device_iommu_mapped(dev)) + err = iommu_probe_device(dev); return err; } -- cgit v1.2.3 From dc9de8a2b20f495696330d60a289935f36407995 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 20 Dec 2018 10:02:20 +0100 Subject: iommu: Check for iommu_ops == NULL in iommu_probe_device() This check needs to be there and got lost at some point during development. Add it again. Fixes: 641fb0efbff0 ('iommu/of: Don't call iommu_ops->add_device directly') Reported-by: Marek Szyprowski Reported-by: kernelci.org bot Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index a2131751dcff..3ed4db334341 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -114,10 +114,14 @@ void iommu_device_unregister(struct iommu_device *iommu) int iommu_probe_device(struct device *dev) { const struct iommu_ops *ops = dev->bus->iommu_ops; + int ret = -EINVAL; WARN_ON(dev->iommu_group); - return ops->add_device(dev); + if (ops) + ret = ops->add_device(dev); + + return ret; } void iommu_release_device(struct device *dev) -- cgit v1.2.3