diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2016-05-20 03:07:04 +0300 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2016-05-20 03:07:04 +0300 |
commit | e0fb1b36398487475e0d2c50264e4ec1eaed3e11 (patch) | |
tree | 4541c8d00a265d2db5b7f2b2c33eb8ccb5819a37 /drivers/iommu/amd_iommu.c | |
parent | f4c80d5a16eb4b08a0d9ade154af1ebdc63f5752 (diff) | |
parent | 6c0b43df74f900e7f31a49d1844f166df0f8afc6 (diff) | |
download | linux-e0fb1b36398487475e0d2c50264e4ec1eaed3e11.tar.xz |
Merge tag 'iommu-updates-v4.7' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu
Pull IOMMU updates from Joerg Roedel:
"The updates include:
- rate limiting for the VT-d fault handler
- remove statistics code from the AMD IOMMU driver. It is unused and
should be replaced by something more generic if needed
- per-domain pagesize-bitmaps in IOMMU core code to support systems
with different types of IOMMUs
- support for ACPI devices in the AMD IOMMU driver
- 4GB mode support for Mediatek IOMMU driver
- ARM-SMMU updates from Will Deacon:
- support for 64k pages with SMMUv1 implementations (e.g MMU-401)
- remove open-coded 64-bit MMIO accessors
- initial support for 16-bit VMIDs, as supported by some ThunderX
SMMU implementations
- a couple of errata workarounds for silicon in the field
- various fixes here and there"
* tag 'iommu-updates-v4.7' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (44 commits)
iommu/arm-smmu: Use per-domain page sizes.
iommu/amd: Remove statistics code
iommu/dma: Finish optimising higher-order allocations
iommu: Allow selecting page sizes per domain
iommu: of: enforce const-ness of struct iommu_ops
iommu: remove unused priv field from struct iommu_ops
iommu/dma: Implement scatterlist segment merging
iommu/arm-smmu: Clear cache lock bit of ACR
iommu/arm-smmu: Support SMMUv1 64KB supplement
iommu/arm-smmu: Decouple context format from kernel config
iommu/arm-smmu: Tidy up 64-bit/atomic I/O accesses
io-64-nonatomic: Add relaxed accessor variants
iommu/arm-smmu: Work around MMU-500 prefetch errata
iommu/arm-smmu: Convert ThunderX workaround to new method
iommu/arm-smmu: Differentiate specific implementations
iommu/arm-smmu: Workaround for ThunderX erratum #27704
iommu/arm-smmu: Add support for 16 bit VMID
iommu/amd: Move get_device_id() and friends to beginning of file
iommu/amd: Don't use IS_ERR_VALUE to check integer values
iommu/amd: Signedness bug in acpihid_device_group()
...
Diffstat (limited to 'drivers/iommu/amd_iommu.c')
-rw-r--r-- | drivers/iommu/amd_iommu.c | 267 |
1 files changed, 149 insertions, 118 deletions
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 3839fd2865a6..634f636393d5 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -19,6 +19,8 @@ #include <linux/ratelimit.h> #include <linux/pci.h> +#include <linux/acpi.h> +#include <linux/amba/bus.h> #include <linux/pci-ats.h> #include <linux/bitmap.h> #include <linux/slab.h> @@ -72,6 +74,7 @@ static DEFINE_SPINLOCK(dev_data_list_lock); LIST_HEAD(ioapic_map); LIST_HEAD(hpet_map); +LIST_HEAD(acpihid_map); /* * Domain for untranslated devices - only allocated @@ -162,18 +165,65 @@ struct dma_ops_domain { * ****************************************************************************/ -static struct protection_domain *to_pdomain(struct iommu_domain *dom) +static inline int match_hid_uid(struct device *dev, + struct acpihid_map_entry *entry) { - return container_of(dom, struct protection_domain, domain); + const char *hid, *uid; + + hid = acpi_device_hid(ACPI_COMPANION(dev)); + uid = acpi_device_uid(ACPI_COMPANION(dev)); + + if (!hid || !(*hid)) + return -ENODEV; + + if (!uid || !(*uid)) + return strcmp(hid, entry->hid); + + if (!(*entry->uid)) + return strcmp(hid, entry->hid); + + return (strcmp(hid, entry->hid) || strcmp(uid, entry->uid)); } -static inline u16 get_device_id(struct device *dev) +static inline u16 get_pci_device_id(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); return PCI_DEVID(pdev->bus->number, pdev->devfn); } +static inline int get_acpihid_device_id(struct device *dev, + struct acpihid_map_entry **entry) +{ + struct acpihid_map_entry *p; + + list_for_each_entry(p, &acpihid_map, list) { + if (!match_hid_uid(dev, p)) { + if (entry) + *entry = p; + return p->devid; + } + } + return -EINVAL; +} + +static inline int get_device_id(struct device *dev) +{ + int devid; + + if (dev_is_pci(dev)) + devid = get_pci_device_id(dev); + else + devid = get_acpihid_device_id(dev, NULL); + + return devid; +} + +static struct protection_domain *to_pdomain(struct iommu_domain *dom) +{ + return container_of(dom, struct protection_domain, domain); +} + static struct iommu_dev_data *alloc_dev_data(u16 devid) { struct iommu_dev_data *dev_data; @@ -222,6 +272,7 @@ static u16 get_alias(struct device *dev) struct pci_dev *pdev = to_pci_dev(dev); u16 devid, ivrs_alias, pci_alias; + /* The callers make sure that get_device_id() does not fail here */ devid = get_device_id(dev); ivrs_alias = amd_iommu_alias_table[devid]; pci_for_each_dma_alias(pdev, __last_alias, &pci_alias); @@ -289,6 +340,29 @@ static struct iommu_dev_data *get_dev_data(struct device *dev) return dev->archdata.iommu; } +/* +* Find or create an IOMMU group for a acpihid device. +*/ +static struct iommu_group *acpihid_device_group(struct device *dev) +{ + struct acpihid_map_entry *p, *entry = NULL; + int devid; + + devid = get_acpihid_device_id(dev, &entry); + if (devid < 0) + return ERR_PTR(devid); + + list_for_each_entry(p, &acpihid_map, list) { + if ((devid == p->devid) && p->group) + entry->group = p->group; + } + + if (!entry->group) + entry->group = generic_device_group(dev); + + return entry->group; +} + static bool pci_iommuv2_capable(struct pci_dev *pdev) { static const int caps[] = { @@ -340,9 +414,11 @@ static void init_unity_mappings_for_device(struct device *dev, struct dma_ops_domain *dma_dom) { struct unity_map_entry *e; - u16 devid; + int devid; devid = get_device_id(dev); + if (devid < 0) + return; list_for_each_entry(e, &amd_iommu_unity_map, list) { if (!(devid >= e->devid_start && devid <= e->devid_end)) @@ -357,16 +433,14 @@ static void init_unity_mappings_for_device(struct device *dev, */ static bool check_device(struct device *dev) { - u16 devid; + int devid; if (!dev || !dev->dma_mask) return false; - /* No PCI device */ - if (!dev_is_pci(dev)) - return false; - devid = get_device_id(dev); + if (devid < 0) + return false; /* Out of our scope? */ if (devid > amd_iommu_last_bdf) @@ -401,22 +475,26 @@ out: static int iommu_init_device(struct device *dev) { - struct pci_dev *pdev = to_pci_dev(dev); struct iommu_dev_data *dev_data; + int devid; if (dev->archdata.iommu) return 0; - dev_data = find_dev_data(get_device_id(dev)); + devid = get_device_id(dev); + if (devid < 0) + return devid; + + dev_data = find_dev_data(devid); if (!dev_data) return -ENOMEM; dev_data->alias = get_alias(dev); - if (pci_iommuv2_capable(pdev)) { + if (dev_is_pci(dev) && pci_iommuv2_capable(to_pci_dev(dev))) { struct amd_iommu *iommu; - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = amd_iommu_rlookup_table[dev_data->devid]; dev_data->iommu_v2 = iommu->is_iommu_v2; } @@ -430,9 +508,13 @@ static int iommu_init_device(struct device *dev) static void iommu_ignore_device(struct device *dev) { - u16 devid, alias; + u16 alias; + int devid; devid = get_device_id(dev); + if (devid < 0) + return; + alias = get_alias(dev); memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry)); @@ -444,8 +526,14 @@ static void iommu_ignore_device(struct device *dev) static void iommu_uninit_device(struct device *dev) { - struct iommu_dev_data *dev_data = search_dev_data(get_device_id(dev)); + int devid; + struct iommu_dev_data *dev_data; + devid = get_device_id(dev); + if (devid < 0) + return; + + dev_data = search_dev_data(devid); if (!dev_data) return; @@ -466,70 +554,6 @@ static void iommu_uninit_device(struct device *dev) */ } -#ifdef CONFIG_AMD_IOMMU_STATS - -/* - * Initialization code for statistics collection - */ - -DECLARE_STATS_COUNTER(compl_wait); -DECLARE_STATS_COUNTER(cnt_map_single); -DECLARE_STATS_COUNTER(cnt_unmap_single); -DECLARE_STATS_COUNTER(cnt_map_sg); -DECLARE_STATS_COUNTER(cnt_unmap_sg); -DECLARE_STATS_COUNTER(cnt_alloc_coherent); -DECLARE_STATS_COUNTER(cnt_free_coherent); -DECLARE_STATS_COUNTER(cross_page); -DECLARE_STATS_COUNTER(domain_flush_single); -DECLARE_STATS_COUNTER(domain_flush_all); -DECLARE_STATS_COUNTER(alloced_io_mem); -DECLARE_STATS_COUNTER(total_map_requests); -DECLARE_STATS_COUNTER(complete_ppr); -DECLARE_STATS_COUNTER(invalidate_iotlb); -DECLARE_STATS_COUNTER(invalidate_iotlb_all); -DECLARE_STATS_COUNTER(pri_requests); - -static struct dentry *stats_dir; -static struct dentry *de_fflush; - -static void amd_iommu_stats_add(struct __iommu_counter *cnt) -{ - if (stats_dir == NULL) - return; - - cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir, - &cnt->value); -} - -static void amd_iommu_stats_init(void) -{ - stats_dir = debugfs_create_dir("amd-iommu", NULL); - if (stats_dir == NULL) - return; - - de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir, - &amd_iommu_unmap_flush); - - amd_iommu_stats_add(&compl_wait); - amd_iommu_stats_add(&cnt_map_single); - amd_iommu_stats_add(&cnt_unmap_single); - amd_iommu_stats_add(&cnt_map_sg); - amd_iommu_stats_add(&cnt_unmap_sg); - amd_iommu_stats_add(&cnt_alloc_coherent); - amd_iommu_stats_add(&cnt_free_coherent); - amd_iommu_stats_add(&cross_page); - amd_iommu_stats_add(&domain_flush_single); - amd_iommu_stats_add(&domain_flush_all); - amd_iommu_stats_add(&alloced_io_mem); - amd_iommu_stats_add(&total_map_requests); - amd_iommu_stats_add(&complete_ppr); - amd_iommu_stats_add(&invalidate_iotlb); - amd_iommu_stats_add(&invalidate_iotlb_all); - amd_iommu_stats_add(&pri_requests); -} - -#endif - /**************************************************************************** * * Interrupt handling functions @@ -652,8 +676,6 @@ static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw) { struct amd_iommu_fault fault; - INC_STATS_COUNTER(pri_requests); - if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) { pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n"); return; @@ -2283,13 +2305,17 @@ static bool pci_pri_tlp_required(struct pci_dev *pdev) static int attach_device(struct device *dev, struct protection_domain *domain) { - struct pci_dev *pdev = to_pci_dev(dev); + struct pci_dev *pdev; struct iommu_dev_data *dev_data; unsigned long flags; int ret; dev_data = get_dev_data(dev); + if (!dev_is_pci(dev)) + goto skip_ats_check; + + pdev = to_pci_dev(dev); if (domain->flags & PD_IOMMUV2_MASK) { if (!dev_data->passthrough) return -EINVAL; @@ -2308,6 +2334,7 @@ static int attach_device(struct device *dev, dev_data->ats.qdep = pci_ats_queue_depth(pdev); } +skip_ats_check: write_lock_irqsave(&amd_iommu_devtable_lock, flags); ret = __attach_device(dev_data, domain); write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); @@ -2364,6 +2391,9 @@ static void detach_device(struct device *dev) __detach_device(dev_data); write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); + if (!dev_is_pci(dev)) + return; + if (domain->flags & PD_IOMMUV2_MASK && dev_data->iommu_v2) pdev_iommuv2_disable(to_pci_dev(dev)); else if (dev_data->ats.enabled) @@ -2377,13 +2407,15 @@ static int amd_iommu_add_device(struct device *dev) struct iommu_dev_data *dev_data; struct iommu_domain *domain; struct amd_iommu *iommu; - u16 devid; - int ret; + int ret, devid; if (!check_device(dev) || get_dev_data(dev)) return 0; devid = get_device_id(dev); + if (devid < 0) + return devid; + iommu = amd_iommu_rlookup_table[devid]; ret = iommu_init_device(dev); @@ -2421,18 +2453,29 @@ out: static void amd_iommu_remove_device(struct device *dev) { struct amd_iommu *iommu; - u16 devid; + int devid; if (!check_device(dev)) return; devid = get_device_id(dev); + if (devid < 0) + return; + iommu = amd_iommu_rlookup_table[devid]; iommu_uninit_device(dev); iommu_completion_wait(iommu); } +static struct iommu_group *amd_iommu_device_group(struct device *dev) +{ + if (dev_is_pci(dev)) + return pci_device_group(dev); + + return acpihid_device_group(dev); +} + /***************************************************************************** * * The next functions belong to the dma_ops mapping/unmapping code. @@ -2597,11 +2640,6 @@ static dma_addr_t __map_single(struct device *dev, pages = iommu_num_pages(paddr, size, PAGE_SIZE); paddr &= PAGE_MASK; - INC_STATS_COUNTER(total_map_requests); - - if (pages > 1) - INC_STATS_COUNTER(cross_page); - if (align) align_mask = (1UL << get_order(size)) - 1; @@ -2622,8 +2660,6 @@ static dma_addr_t __map_single(struct device *dev, } address += offset; - ADD_STATS_COUNTER(alloced_io_mem, size); - if (unlikely(amd_iommu_np_cache)) { domain_flush_pages(&dma_dom->domain, address, size); domain_flush_complete(&dma_dom->domain); @@ -2671,8 +2707,6 @@ static void __unmap_single(struct dma_ops_domain *dma_dom, start += PAGE_SIZE; } - SUB_STATS_COUNTER(alloced_io_mem, size); - dma_ops_free_addresses(dma_dom, dma_addr, pages); } @@ -2688,8 +2722,6 @@ static dma_addr_t map_page(struct device *dev, struct page *page, struct protection_domain *domain; u64 dma_mask; - INC_STATS_COUNTER(cnt_map_single); - domain = get_domain(dev); if (PTR_ERR(domain) == -EINVAL) return (dma_addr_t)paddr; @@ -2710,8 +2742,6 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, { struct protection_domain *domain; - INC_STATS_COUNTER(cnt_unmap_single); - domain = get_domain(dev); if (IS_ERR(domain)) return; @@ -2734,8 +2764,6 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, int mapped_elems = 0; u64 dma_mask; - INC_STATS_COUNTER(cnt_map_sg); - domain = get_domain(dev); if (IS_ERR(domain)) return 0; @@ -2781,8 +2809,6 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist, struct scatterlist *s; int i; - INC_STATS_COUNTER(cnt_unmap_sg); - domain = get_domain(dev); if (IS_ERR(domain)) return; @@ -2805,8 +2831,6 @@ static void *alloc_coherent(struct device *dev, size_t size, struct protection_domain *domain; struct page *page; - INC_STATS_COUNTER(cnt_alloc_coherent); - domain = get_domain(dev); if (PTR_ERR(domain) == -EINVAL) { page = alloc_pages(flag, get_order(size)); @@ -2860,8 +2884,6 @@ static void free_coherent(struct device *dev, size_t size, struct protection_domain *domain; struct page *page; - INC_STATS_COUNTER(cnt_free_coherent); - page = virt_to_page(virt_addr); size = PAGE_ALIGN(size); @@ -2926,7 +2948,17 @@ static struct dma_map_ops amd_iommu_dma_ops = { int __init amd_iommu_init_api(void) { - return bus_set_iommu(&pci_bus_type, &amd_iommu_ops); + int err = 0; + + err = bus_set_iommu(&pci_bus_type, &amd_iommu_ops); + if (err) + return err; +#ifdef CONFIG_ARM_AMBA + err = bus_set_iommu(&amba_bustype, &amd_iommu_ops); + if (err) + return err; +#endif + return 0; } int __init amd_iommu_init_dma_ops(void) @@ -2943,8 +2975,6 @@ int __init amd_iommu_init_dma_ops(void) if (!swiotlb) dma_ops = &nommu_dma_ops; - amd_iommu_stats_init(); - if (amd_iommu_unmap_flush) pr_info("AMD-Vi: IO/TLB flush on unmap enabled\n"); else @@ -3098,12 +3128,14 @@ static void amd_iommu_detach_device(struct iommu_domain *dom, { struct iommu_dev_data *dev_data = dev->archdata.iommu; struct amd_iommu *iommu; - u16 devid; + int devid; if (!check_device(dev)) return; devid = get_device_id(dev); + if (devid < 0) + return; if (dev_data->domain != NULL) detach_device(dev); @@ -3221,9 +3253,11 @@ static void amd_iommu_get_dm_regions(struct device *dev, struct list_head *head) { struct unity_map_entry *entry; - u16 devid; + int devid; devid = get_device_id(dev); + if (devid < 0) + return; list_for_each_entry(entry, &amd_iommu_unity_map, list) { struct iommu_dm_region *region; @@ -3270,7 +3304,7 @@ static const struct iommu_ops amd_iommu_ops = { .iova_to_phys = amd_iommu_iova_to_phys, .add_device = amd_iommu_add_device, .remove_device = amd_iommu_remove_device, - .device_group = pci_device_group, + .device_group = amd_iommu_device_group, .get_dm_regions = amd_iommu_get_dm_regions, .put_dm_regions = amd_iommu_put_dm_regions, .pgsize_bitmap = AMD_IOMMU_PGSIZES, @@ -3431,8 +3465,6 @@ out: static int __amd_iommu_flush_page(struct protection_domain *domain, int pasid, u64 address) { - INC_STATS_COUNTER(invalidate_iotlb); - return __flush_pasid(domain, pasid, address, false); } @@ -3453,8 +3485,6 @@ EXPORT_SYMBOL(amd_iommu_flush_page); static int __amd_iommu_flush_tlb(struct protection_domain *domain, int pasid) { - INC_STATS_COUNTER(invalidate_iotlb_all); - return __flush_pasid(domain, pasid, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, true); } @@ -3574,8 +3604,6 @@ int amd_iommu_complete_ppr(struct pci_dev *pdev, int pasid, struct amd_iommu *iommu; struct iommu_cmd cmd; - INC_STATS_COUNTER(complete_ppr); - dev_data = get_dev_data(&pdev->dev); iommu = amd_iommu_rlookup_table[dev_data->devid]; @@ -3925,6 +3953,9 @@ static struct irq_domain *get_irq_domain(struct irq_alloc_info *info) case X86_IRQ_ALLOC_TYPE_MSI: case X86_IRQ_ALLOC_TYPE_MSIX: devid = get_device_id(&info->msi_dev->dev); + if (devid < 0) + return NULL; + iommu = amd_iommu_rlookup_table[devid]; if (iommu) return iommu->msi_domain; |