diff options
Diffstat (limited to 'drivers/iommu')
27 files changed, 477 insertions, 626 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index d9a25715650e..6f07f3b21816 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -1,3 +1,7 @@ +# The IOVA library may also be used by non-IOMMU_API users +config IOMMU_IOVA + tristate + # IOMMU_API always gets selected by whoever wants it. config IOMMU_API bool @@ -81,9 +85,6 @@ config IOMMU_DEFAULT_PASSTHROUGH If unsure, say N here. -config IOMMU_IOVA - tristate - config OF_IOMMU def_bool y depends on OF && IOMMU_API @@ -282,6 +283,7 @@ config ROCKCHIP_IOMMU config TEGRA_IOMMU_GART bool "Tegra GART IOMMU Support" depends on ARCH_TEGRA_2x_SOC + depends on TEGRA_MC select IOMMU_API help Enables support for remapping discontiguous physical memory @@ -435,4 +437,13 @@ config QCOM_IOMMU help Support for IOMMU on certain Qualcomm SoCs. +config HYPERV_IOMMU + bool "Hyper-V x2APIC IRQ Handling" + depends on HYPERV + select IOMMU_API + default HYPERV + help + Stub IOMMU driver to handle IRQs as to allow Hyper-V Linux + guests to run with x2APIC mode enabled. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index a158a68c8ea8..8c71a15e986b 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -32,3 +32,4 @@ obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o obj-$(CONFIG_S390_IOMMU) += s390-iommu.o obj-$(CONFIG_QCOM_IOMMU) += qcom_iommu.o +obj-$(CONFIG_HYPERV_IOMMU) += hyperv-iommu.o diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 87ba23a75b38..6b0760dafb3e 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -18,6 +18,7 @@ */ #define pr_fmt(fmt) "AMD-Vi: " fmt +#define dev_fmt(fmt) pr_fmt(fmt) #include <linux/ratelimit.h> #include <linux/pci.h> @@ -279,10 +280,10 @@ static u16 get_alias(struct device *dev) return pci_alias; } - pr_info("Using IVRS reported alias %02x:%02x.%d " - "for device %s[%04x:%04x], kernel reported alias " + pci_info(pdev, "Using IVRS reported alias %02x:%02x.%d " + "for device [%04x:%04x], kernel reported alias " "%02x:%02x.%d\n", PCI_BUS_NUM(ivrs_alias), PCI_SLOT(ivrs_alias), - PCI_FUNC(ivrs_alias), dev_name(dev), pdev->vendor, pdev->device, + PCI_FUNC(ivrs_alias), pdev->vendor, pdev->device, PCI_BUS_NUM(pci_alias), PCI_SLOT(pci_alias), PCI_FUNC(pci_alias)); @@ -293,9 +294,8 @@ static u16 get_alias(struct device *dev) if (pci_alias == devid && PCI_BUS_NUM(ivrs_alias) == pdev->bus->number) { pci_add_dma_alias(pdev, ivrs_alias & 0xff); - pr_info("Added PCI DMA alias %02x.%d for %s\n", - PCI_SLOT(ivrs_alias), PCI_FUNC(ivrs_alias), - dev_name(dev)); + pci_info(pdev, "Added PCI DMA alias %02x.%d\n", + PCI_SLOT(ivrs_alias), PCI_FUNC(ivrs_alias)); } return ivrs_alias; @@ -545,7 +545,7 @@ static void amd_iommu_report_page_fault(u16 devid, u16 domain_id, dev_data = get_dev_data(&pdev->dev); if (dev_data && __ratelimit(&dev_data->rs)) { - dev_err(&pdev->dev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%llx flags=0x%04x]\n", + pci_err(pdev, "Event logged [IO_PAGE_FAULT domain=0x%04x address=0x%llx flags=0x%04x]\n", domain_id, address, flags); } else if (printk_ratelimit()) { pr_err("Event logged [IO_PAGE_FAULT device=%02x:%02x.%x domain=0x%04x address=0x%llx flags=0x%04x]\n", @@ -1991,16 +1991,13 @@ static void do_attach(struct iommu_dev_data *dev_data, static void do_detach(struct iommu_dev_data *dev_data) { + struct protection_domain *domain = dev_data->domain; struct amd_iommu *iommu; u16 alias; iommu = amd_iommu_rlookup_table[dev_data->devid]; alias = dev_data->alias; - /* decrease reference counters */ - dev_data->domain->dev_iommu[iommu->index] -= 1; - dev_data->domain->dev_cnt -= 1; - /* Update data structures */ dev_data->domain = NULL; list_del(&dev_data->list); @@ -2010,6 +2007,16 @@ static void do_detach(struct iommu_dev_data *dev_data) /* Flush the DTE entry */ device_flush_dte(dev_data); + + /* Flush IOTLB */ + domain_flush_tlb_pde(domain); + + /* Wait for the flushes to finish */ + domain_flush_complete(domain); + + /* decrease reference counters - needs to happen after the flushes */ + domain->dev_iommu[iommu->index] -= 1; + domain->dev_cnt -= 1; } /* @@ -2251,8 +2258,7 @@ static int amd_iommu_add_device(struct device *dev) ret = iommu_init_device(dev); if (ret) { if (ret != -ENOTSUPP) - pr_err("Failed to initialize device %s - trying to proceed anyway\n", - dev_name(dev)); + dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); iommu_ignore_device(dev); dev->dma_ops = NULL; @@ -2562,6 +2568,7 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, struct scatterlist *s; unsigned long address; u64 dma_mask; + int ret; domain = get_domain(dev); if (IS_ERR(domain)) @@ -2584,7 +2591,6 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, for (j = 0; j < pages; ++j) { unsigned long bus_addr, phys_addr; - int ret; bus_addr = address + s->dma_address + (j << PAGE_SHIFT); phys_addr = (sg_phys(s) & PAGE_MASK) + (j << PAGE_SHIFT); @@ -2605,8 +2611,8 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, return nelems; out_unmap: - pr_err("%s: IOMMU mapping error in map_sg (io-pages: %d)\n", - dev_name(dev), npages); + dev_err(dev, "IOMMU mapping error in map_sg (io-pages: %d reason: %d)\n", + npages, ret); for_each_sg(sglist, s, nelems, i) { int j, pages = iommu_num_pages(sg_phys(s), s->length, PAGE_SIZE); @@ -2617,13 +2623,13 @@ out_unmap: bus_addr = address + s->dma_address + (j << PAGE_SHIFT); iommu_unmap_page(domain, bus_addr, PAGE_SIZE); - if (--mapped_pages) + if (--mapped_pages == 0) goto out_free_iova; } } out_free_iova: - free_iova_fast(&dma_dom->iovad, address, npages); + free_iova_fast(&dma_dom->iovad, address >> PAGE_SHIFT, npages); out_err: return 0; @@ -2800,7 +2806,7 @@ static int init_reserved_iova_ranges(void) IOVA_PFN(r->start), IOVA_PFN(r->end)); if (!val) { - pr_err("Reserve pci-resource range failed\n"); + pci_err(pdev, "Reserve pci-resource range %pR failed\n", r); return -ENOMEM; } } @@ -3170,8 +3176,7 @@ static void amd_iommu_get_resv_regions(struct device *dev, length, prot, IOMMU_RESV_DIRECT); if (!region) { - pr_err("Out of memory allocating dm-regions for %s\n", - dev_name(dev)); + dev_err(dev, "Out of memory allocating dm-regions\n"); return; } list_add_tail(®ion->list, head); diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 66123b911ec8..f773792d77fd 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -18,6 +18,7 @@ */ #define pr_fmt(fmt) "AMD-Vi: " fmt +#define dev_fmt(fmt) pr_fmt(fmt) #include <linux/pci.h> #include <linux/acpi.h> @@ -1457,8 +1458,7 @@ static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu) pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8)); pci_write_config_dword(iommu->dev, 0xf4, value | 0x4); - pr_info("Applying erratum 746 workaround for IOMMU at %s\n", - dev_name(&iommu->dev->dev)); + pci_info(iommu->dev, "Applying erratum 746 workaround\n"); /* Clear the enable writing bit */ pci_write_config_dword(iommu->dev, 0xf0, 0x90); @@ -1488,8 +1488,7 @@ static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu) /* Set L2_DEBUG_3[AtsIgnoreIWDis] = 1 */ iommu_write_l2(iommu, 0x47, value | BIT(0)); - pr_info("Applying ATS write check workaround for IOMMU at %s\n", - dev_name(&iommu->dev->dev)); + pci_info(iommu->dev, "Applying ATS write check workaround\n"); } /* @@ -1665,6 +1664,7 @@ static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, static void init_iommu_perf_ctr(struct amd_iommu *iommu) { + struct pci_dev *pdev = iommu->dev; u64 val = 0xabcd, val2 = 0; if (!iommu_feature(iommu, FEATURE_PC)) @@ -1676,12 +1676,12 @@ static void init_iommu_perf_ctr(struct amd_iommu *iommu) if ((iommu_pc_get_set_reg(iommu, 0, 0, 0, &val, true)) || (iommu_pc_get_set_reg(iommu, 0, 0, 0, &val2, false)) || (val != val2)) { - pr_err("Unable to write to IOMMU perf counter.\n"); + pci_err(pdev, "Unable to write to IOMMU perf counter.\n"); amd_iommu_pc_present = false; return; } - pr_info("IOMMU performance counters supported\n"); + pci_info(pdev, "IOMMU performance counters supported\n"); val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET); iommu->max_banks = (u8) ((val >> 12) & 0x3f); @@ -1840,14 +1840,14 @@ static void print_iommu_info(void) struct amd_iommu *iommu; for_each_iommu(iommu) { + struct pci_dev *pdev = iommu->dev; int i; - pr_info("Found IOMMU at %s cap 0x%hx\n", - dev_name(&iommu->dev->dev), iommu->cap_ptr); + pci_info(pdev, "Found IOMMU cap 0x%hx\n", iommu->cap_ptr); if (iommu->cap & (1 << IOMMU_CAP_EFR)) { - pr_info("Extended features (%#llx):\n", - iommu->features); + pci_info(pdev, "Extended features (%#llx):\n", + iommu->features); for (i = 0; i < ARRAY_SIZE(feat_str); ++i) { if (iommu_feature(iommu, (1ULL << i))) pr_cont(" %s", feat_str[i]); diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 23dae9348ace..5d7ef750e4a0 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -370,29 +370,6 @@ static struct pasid_state *mn_to_state(struct mmu_notifier *mn) return container_of(mn, struct pasid_state, mn); } -static void __mn_flush_page(struct mmu_notifier *mn, - unsigned long address) -{ - struct pasid_state *pasid_state; - struct device_state *dev_state; - - pasid_state = mn_to_state(mn); - dev_state = pasid_state->device_state; - - amd_iommu_flush_page(dev_state->domain, pasid_state->pasid, address); -} - -static int mn_clear_flush_young(struct mmu_notifier *mn, - struct mm_struct *mm, - unsigned long start, - unsigned long end) -{ - for (; start < end; start += PAGE_SIZE) - __mn_flush_page(mn, start); - - return 0; -} - static void mn_invalidate_range(struct mmu_notifier *mn, struct mm_struct *mm, unsigned long start, unsigned long end) @@ -430,7 +407,6 @@ static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm) static const struct mmu_notifier_ops iommu_mn = { .release = mn_release, - .clear_flush_young = mn_clear_flush_young, .invalidate_range = mn_invalidate_range, }; diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 0d284029dc73..d3880010c6cf 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -18,6 +18,7 @@ #include <linux/dma-iommu.h> #include <linux/err.h> #include <linux/interrupt.h> +#include <linux/io-pgtable.h> #include <linux/iommu.h> #include <linux/iopoll.h> #include <linux/init.h> @@ -32,8 +33,6 @@ #include <linux/amba/bus.h> -#include "io-pgtable.h" - /* MMIO registers */ #define ARM_SMMU_IDR0 0x0 #define IDR0_ST_LVL GENMASK(28, 27) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index af18a7e7f917..045d93884164 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -39,6 +39,7 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/io-64-nonatomic-hi-lo.h> +#include <linux/io-pgtable.h> #include <linux/iommu.h> #include <linux/iopoll.h> #include <linux/init.h> @@ -56,7 +57,6 @@ #include <linux/amba/bus.h> #include <linux/fsl/mc.h> -#include "io-pgtable.h" #include "arm-smmu-regs.h" #define ARM_MMU500_ACTLR_CPRE (1 << 1) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index d19f3d6b43c1..77aabe637a60 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -289,7 +289,7 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, { struct iommu_dma_cookie *cookie = domain->iova_cookie; struct iova_domain *iovad = &cookie->iovad; - unsigned long order, base_pfn, end_pfn; + unsigned long order, base_pfn; int attr; if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE) @@ -298,7 +298,6 @@ int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base, /* Use the smallest supported page size for IOVA granularity */ order = __ffs(domain->pgsize_bitmap); base_pfn = max_t(unsigned long, 1, base >> order); - end_pfn = (base + size - 1) >> order; /* Check the domain allows at least some access to the device... */ if (domain->geometry.force_aperture) { diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index dc9f14811e0f..58dc70bffd5b 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -144,7 +144,7 @@ dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event) for (tmp = dev; tmp; tmp = tmp->bus->self) level++; - size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path); + size = sizeof(*info) + level * sizeof(info->path[0]); if (size <= sizeof(dmar_pci_notify_info_buf)) { info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf; } else { diff --git a/drivers/iommu/hyperv-iommu.c b/drivers/iommu/hyperv-iommu.c new file mode 100644 index 000000000000..a386b83e0e34 --- /dev/null +++ b/drivers/iommu/hyperv-iommu.c @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Hyper-V stub IOMMU driver. + * + * Copyright (C) 2019, Microsoft, Inc. + * + * Author : Lan Tianyu <Tianyu.Lan@microsoft.com> + */ + +#include <linux/types.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/iommu.h> +#include <linux/module.h> + +#include <asm/apic.h> +#include <asm/cpu.h> +#include <asm/hw_irq.h> +#include <asm/io_apic.h> +#include <asm/irq_remapping.h> +#include <asm/hypervisor.h> + +#include "irq_remapping.h" + +#ifdef CONFIG_IRQ_REMAP + +/* + * According 82093AA IO-APIC spec , IO APIC has a 24-entry Interrupt + * Redirection Table. Hyper-V exposes one single IO-APIC and so define + * 24 IO APIC remmapping entries. + */ +#define IOAPIC_REMAPPING_ENTRY 24 + +static cpumask_t ioapic_max_cpumask = { CPU_BITS_NONE }; +static struct irq_domain *ioapic_ir_domain; + +static int hyperv_ir_set_affinity(struct irq_data *data, + const struct cpumask *mask, bool force) +{ + struct irq_data *parent = data->parent_data; + struct irq_cfg *cfg = irqd_cfg(data); + struct IO_APIC_route_entry *entry; + int ret; + + /* Return error If new irq affinity is out of ioapic_max_cpumask. */ + if (!cpumask_subset(mask, &ioapic_max_cpumask)) + return -EINVAL; + + ret = parent->chip->irq_set_affinity(parent, mask, force); + if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) + return ret; + + entry = data->chip_data; + entry->dest = cfg->dest_apicid; + entry->vector = cfg->vector; + send_cleanup_vector(cfg); + + return 0; +} + +static struct irq_chip hyperv_ir_chip = { + .name = "HYPERV-IR", + .irq_ack = apic_ack_irq, + .irq_set_affinity = hyperv_ir_set_affinity, +}; + +static int hyperv_irq_remapping_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *arg) +{ + struct irq_alloc_info *info = arg; + struct irq_data *irq_data; + struct irq_desc *desc; + int ret = 0; + + if (!info || info->type != X86_IRQ_ALLOC_TYPE_IOAPIC || nr_irqs > 1) + return -EINVAL; + + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); + if (ret < 0) + return ret; + + irq_data = irq_domain_get_irq_data(domain, virq); + if (!irq_data) { + irq_domain_free_irqs_common(domain, virq, nr_irqs); + return -EINVAL; + } + + irq_data->chip = &hyperv_ir_chip; + + /* + * If there is interrupt remapping function of IOMMU, setting irq + * affinity only needs to change IRTE of IOMMU. But Hyper-V doesn't + * support interrupt remapping function, setting irq affinity of IO-APIC + * interrupts still needs to change IO-APIC registers. But ioapic_ + * configure_entry() will ignore value of cfg->vector and cfg-> + * dest_apicid when IO-APIC's parent irq domain is not the vector + * domain.(See ioapic_configure_entry()) In order to setting vector + * and dest_apicid to IO-APIC register, IO-APIC entry pointer is saved + * in the chip_data and hyperv_irq_remapping_activate()/hyperv_ir_set_ + * affinity() set vector and dest_apicid directly into IO-APIC entry. + */ + irq_data->chip_data = info->ioapic_entry; + + /* + * Hypver-V IO APIC irq affinity should be in the scope of + * ioapic_max_cpumask because no irq remapping support. + */ + desc = irq_data_to_desc(irq_data); + cpumask_copy(desc->irq_common_data.affinity, &ioapic_max_cpumask); + + return 0; +} + +static void hyperv_irq_remapping_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + irq_domain_free_irqs_common(domain, virq, nr_irqs); +} + +static int hyperv_irq_remapping_activate(struct irq_domain *domain, + struct irq_data *irq_data, bool reserve) +{ + struct irq_cfg *cfg = irqd_cfg(irq_data); + struct IO_APIC_route_entry *entry = irq_data->chip_data; + + entry->dest = cfg->dest_apicid; + entry->vector = cfg->vector; + + return 0; +} + +static struct irq_domain_ops hyperv_ir_domain_ops = { + .alloc = hyperv_irq_remapping_alloc, + .free = hyperv_irq_remapping_free, + .activate = hyperv_irq_remapping_activate, +}; + +static int __init hyperv_prepare_irq_remapping(void) +{ + struct fwnode_handle *fn; + int i; + + if (!hypervisor_is_type(X86_HYPER_MS_HYPERV) || + !x2apic_supported()) + return -ENODEV; + + fn = irq_domain_alloc_named_id_fwnode("HYPERV-IR", 0); + if (!fn) + return -ENOMEM; + + ioapic_ir_domain = + irq_domain_create_hierarchy(arch_get_ir_parent_domain(), + 0, IOAPIC_REMAPPING_ENTRY, fn, + &hyperv_ir_domain_ops, NULL); + + irq_domain_free_fwnode(fn); + + /* + * Hyper-V doesn't provide irq remapping function for + * IO-APIC and so IO-APIC only accepts 8-bit APIC ID. + * Cpu's APIC ID is read from ACPI MADT table and APIC IDs + * in the MADT table on Hyper-v are sorted monotonic increasingly. + * APIC ID reflects cpu topology. There maybe some APIC ID + * gaps when cpu number in a socket is not power of two. Prepare + * max cpu affinity for IOAPIC irqs. Scan cpu 0-255 and set cpu + * into ioapic_max_cpumask if its APIC ID is less than 256. + */ + for (i = min_t(unsigned int, num_possible_cpus() - 1, 255); i >= 0; i--) + if (cpu_physical_id(i) < 256) + cpumask_set_cpu(i, &ioapic_max_cpumask); + + return 0; +} + +static int __init hyperv_enable_irq_remapping(void) +{ + return IRQ_REMAP_X2APIC_MODE; +} + +static struct irq_domain *hyperv_get_ir_irq_domain(struct irq_alloc_info *info) +{ + if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) + return ioapic_ir_domain; + else + return NULL; +} + +struct irq_remap_ops hyperv_irq_remap_ops = { + .prepare = hyperv_prepare_irq_remapping, + .enable = hyperv_enable_irq_remapping, + .get_ir_irq_domain = hyperv_get_ir_irq_domain, +}; + +#endif diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index d7f10fd4ac2d..c968b3c7bae0 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -363,7 +363,7 @@ static int dmar_map_gfx = 1; static int dmar_forcedac; static int intel_iommu_strict; static int intel_iommu_superpage = 1; -static int intel_iommu_sm = 1; +static int intel_iommu_sm; static int iommu_identity_mapping; #define IDENTMAP_ALL 1 @@ -456,9 +456,9 @@ static int __init intel_iommu_setup(char *str) } else if (!strncmp(str, "sp_off", 6)) { pr_info("Disable supported super page\n"); intel_iommu_superpage = 0; - } else if (!strncmp(str, "sm_off", 6)) { - pr_info("Intel-IOMMU: disable scalable mode support\n"); - intel_iommu_sm = 0; + } else if (!strncmp(str, "sm_on", 5)) { + pr_info("Intel-IOMMU: scalable mode supported\n"); + intel_iommu_sm = 1; } else if (!strncmp(str, "tboot_noforce", 13)) { printk(KERN_INFO "Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n"); @@ -5298,7 +5298,7 @@ static void intel_iommu_put_resv_regions(struct device *dev, struct iommu_resv_region *entry, *next; list_for_each_entry_safe(entry, next, head, list) { - if (entry->type == IOMMU_RESV_RESERVED) + if (entry->type == IOMMU_RESV_MSI) kfree(entry); } } diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index cec29bf45c9b..f101afc315ab 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -35,6 +35,7 @@ #include <linux/atomic.h> #include <linux/dma-mapping.h> #include <linux/gfp.h> +#include <linux/io-pgtable.h> #include <linux/iommu.h> #include <linux/kernel.h> #include <linux/kmemleak.h> @@ -45,8 +46,6 @@ #include <asm/barrier.h> -#include "io-pgtable.h" - /* Struct accessors */ #define io_pgtable_to_data(x) \ container_of((x), struct arm_v7s_io_pgtable, iop) @@ -217,7 +216,8 @@ static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp, if (dma != phys) goto out_unmap; } - kmemleak_ignore(table); + if (lvl == 2) + kmemleak_ignore(table); return table; out_unmap: diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 237cacd4a62b..d3700ec15cbd 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -22,6 +22,7 @@ #include <linux/atomic.h> #include <linux/bitops.h> +#include <linux/io-pgtable.h> #include <linux/iommu.h> #include <linux/kernel.h> #include <linux/sizes.h> @@ -31,8 +32,6 @@ #include <asm/barrier.h> -#include "io-pgtable.h" - #define ARM_LPAE_MAX_ADDR_BITS 52 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 #define ARM_LPAE_MAX_LEVELS 4 diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index 127558d83667..93f2880be6c6 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -19,11 +19,10 @@ */ #include <linux/bug.h> +#include <linux/io-pgtable.h> #include <linux/kernel.h> #include <linux/types.h> -#include "io-pgtable.h" - static const struct io_pgtable_init_fns * io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { #ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE @@ -61,6 +60,7 @@ struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, return &iop->ops; } +EXPORT_SYMBOL_GPL(alloc_io_pgtable_ops); /* * It is the IOMMU driver's responsibility to ensure that the page table @@ -77,3 +77,4 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops) io_pgtable_tlb_flush_all(iop); io_pgtable_init_table[iop->fmt]->free(iop); } +EXPORT_SYMBOL_GPL(free_io_pgtable_ops); diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h deleted file mode 100644 index 47d5ae559329..000000000000 --- a/drivers/iommu/io-pgtable.h +++ /dev/null @@ -1,213 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __IO_PGTABLE_H -#define __IO_PGTABLE_H -#include <linux/bitops.h> - -/* - * Public API for use by IOMMU drivers - */ -enum io_pgtable_fmt { - ARM_32_LPAE_S1, - ARM_32_LPAE_S2, - ARM_64_LPAE_S1, - ARM_64_LPAE_S2, - ARM_V7S, - IO_PGTABLE_NUM_FMTS, -}; - -/** - * struct iommu_gather_ops - IOMMU callbacks for TLB and page table management. - * - * @tlb_flush_all: Synchronously invalidate the entire TLB context. - * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range. - * @tlb_sync: Ensure any queued TLB invalidation has taken effect, and - * any corresponding page table updates are visible to the - * IOMMU. - * - * Note that these can all be called in atomic context and must therefore - * not block. - */ -struct iommu_gather_ops { - void (*tlb_flush_all)(void *cookie); - void (*tlb_add_flush)(unsigned long iova, size_t size, size_t granule, - bool leaf, void *cookie); - void (*tlb_sync)(void *cookie); -}; - -/** - * struct io_pgtable_cfg - Configuration data for a set of page tables. - * - * @quirks: A bitmap of hardware quirks that require some special - * action by the low-level page table allocator. - * @pgsize_bitmap: A bitmap of page sizes supported by this set of page - * tables. - * @ias: Input address (iova) size, in bits. - * @oas: Output address (paddr) size, in bits. - * @tlb: TLB management callbacks for this set of tables. - * @iommu_dev: The device representing the DMA configuration for the - * page table walker. - */ -struct io_pgtable_cfg { - /* - * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in - * stage 1 PTEs, for hardware which insists on validating them - * even in non-secure state where they should normally be ignored. - * - * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and - * IOMMU_NOEXEC flags and map everything with full access, for - * hardware which does not implement the permissions of a given - * format, and/or requires some format-specific default value. - * - * IO_PGTABLE_QUIRK_TLBI_ON_MAP: If the format forbids caching invalid - * (unmapped) entries but the hardware might do so anyway, perform - * TLB maintenance when mapping as well as when unmapping. - * - * IO_PGTABLE_QUIRK_ARM_MTK_4GB: (ARM v7s format) Set bit 9 in all - * PTEs, for Mediatek IOMMUs which treat it as a 33rd address bit - * when the SoC is in "4GB mode" and they can only access the high - * remap of DRAM (0x1_00000000 to 0x1_ffffffff). - * - * IO_PGTABLE_QUIRK_NO_DMA: Guarantees that the tables will only ever - * be accessed by a fully cache-coherent IOMMU or CPU (e.g. for a - * software-emulated IOMMU), such that pagetable updates need not - * be treated as explicit DMA data. - * - * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs - * on unmap, for DMA domains using the flush queue mechanism for - * delayed invalidation. - */ - #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) - #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) - #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2) - #define IO_PGTABLE_QUIRK_ARM_MTK_4GB BIT(3) - #define IO_PGTABLE_QUIRK_NO_DMA BIT(4) - #define IO_PGTABLE_QUIRK_NON_STRICT BIT(5) - unsigned long quirks; - unsigned long pgsize_bitmap; - unsigned int ias; - unsigned int oas; - const struct iommu_gather_ops *tlb; - struct device *iommu_dev; - - /* Low-level data specific to the table format */ - union { - struct { - u64 ttbr[2]; - u64 tcr; - u64 mair[2]; - } arm_lpae_s1_cfg; - - struct { - u64 vttbr; - u64 vtcr; - } arm_lpae_s2_cfg; - - struct { - u32 ttbr[2]; - u32 tcr; - u32 nmrr; - u32 prrr; - } arm_v7s_cfg; - }; -}; - -/** - * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers. - * - * @map: Map a physically contiguous memory region. - * @unmap: Unmap a physically contiguous memory region. - * @iova_to_phys: Translate iova to physical address. - * - * These functions map directly onto the iommu_ops member functions with - * the same names. - */ -struct io_pgtable_ops { - int (*map)(struct io_pgtable_ops *ops, unsigned long iova, - phys_addr_t paddr, size_t size, int prot); - size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova, - size_t size); - phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops, - unsigned long iova); -}; - -/** - * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU. - * - * @fmt: The page table format. - * @cfg: The page table configuration. This will be modified to represent - * the configuration actually provided by the allocator (e.g. the - * pgsize_bitmap may be restricted). - * @cookie: An opaque token provided by the IOMMU driver and passed back to - * the callback routines in cfg->tlb. - */ -struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, - struct io_pgtable_cfg *cfg, - void *cookie); - -/** - * free_io_pgtable_ops() - Free an io_pgtable_ops structure. The caller - * *must* ensure that the page table is no longer - * live, but the TLB can be dirty. - * - * @ops: The ops returned from alloc_io_pgtable_ops. - */ -void free_io_pgtable_ops(struct io_pgtable_ops *ops); - - -/* - * Internal structures for page table allocator implementations. - */ - -/** - * struct io_pgtable - Internal structure describing a set of page tables. - * - * @fmt: The page table format. - * @cookie: An opaque token provided by the IOMMU driver and passed back to - * any callback routines. - * @cfg: A copy of the page table configuration. - * @ops: The page table operations in use for this set of page tables. - */ -struct io_pgtable { - enum io_pgtable_fmt fmt; - void *cookie; - struct io_pgtable_cfg cfg; - struct io_pgtable_ops ops; -}; - -#define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops) - -static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop) -{ - iop->cfg.tlb->tlb_flush_all(iop->cookie); -} - -static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop, - unsigned long iova, size_t size, size_t granule, bool leaf) -{ - iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie); -} - -static inline void io_pgtable_tlb_sync(struct io_pgtable *iop) -{ - iop->cfg.tlb->tlb_sync(iop->cookie); -} - -/** - * struct io_pgtable_init_fns - Alloc/free a set of page tables for a - * particular format. - * - * @alloc: Allocate a set of page tables described by cfg. - * @free: Free the page tables associated with iop. - */ -struct io_pgtable_init_fns { - struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie); - void (*free)(struct io_pgtable *iop); -}; - -extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; -extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; -extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; -extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; -extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns; - -#endif /* __IO_PGTABLE_H */ diff --git a/drivers/iommu/iommu-debugfs.c b/drivers/iommu/iommu-debugfs.c index 3b1bf88fd1b0..f03548942096 100644 --- a/drivers/iommu/iommu-debugfs.c +++ b/drivers/iommu/iommu-debugfs.c @@ -12,6 +12,7 @@ #include <linux/debugfs.h> struct dentry *iommu_debugfs_dir; +EXPORT_SYMBOL_GPL(iommu_debugfs_dir); /** * iommu_debugfs_setup - create the top-level iommu directory in debugfs @@ -23,9 +24,9 @@ struct dentry *iommu_debugfs_dir; * Emit a strong warning at boot time to indicate that this feature is * enabled. * - * This function is called from iommu_init; drivers may then call - * iommu_debugfs_new_driver_dir() to instantiate a vendor-specific - * directory to be used to expose internal data. + * This function is called from iommu_init; drivers may then use + * iommu_debugfs_dir to instantiate a vendor-specific directory to be used + * to expose internal data. */ void iommu_debugfs_setup(void) { @@ -48,19 +49,3 @@ void iommu_debugfs_setup(void) pr_warn("*************************************************************\n"); } } - -/** - * iommu_debugfs_new_driver_dir - create a vendor directory under debugfs/iommu - * @vendor: name of the vendor-specific subdirectory to create - * - * This function is called by an IOMMU driver to create the top-level debugfs - * directory for that driver. - * - * Return: upon success, a pointer to the dentry for the new directory. - * NULL in case of failure. - */ -struct dentry *iommu_debugfs_new_driver_dir(const char *vendor) -{ - return debugfs_create_dir(vendor, iommu_debugfs_dir); -} -EXPORT_SYMBOL_GPL(iommu_debugfs_new_driver_dir); diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 3ed4db334341..33a982e33716 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -668,7 +668,7 @@ rename: trace_add_device_to_group(group->id, dev); - pr_info("Adding device %s to group %d\n", dev_name(dev), group->id); + dev_info(dev, "Adding to iommu group %d\n", group->id); return 0; @@ -684,7 +684,7 @@ err_remove_link: sysfs_remove_link(&dev->kobj, "iommu_group"); err_free_device: kfree(device); - pr_err("Failed to add device %s to group %d: %d\n", dev_name(dev), group->id, ret); + dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret); return ret; } EXPORT_SYMBOL_GPL(iommu_group_add_device); @@ -701,7 +701,7 @@ void iommu_group_remove_device(struct device *dev) struct iommu_group *group = dev->iommu_group; struct group_device *tmp_device, *device = NULL; - pr_info("Removing device %s from group %d\n", dev_name(dev), group->id); + dev_info(dev, "Removing from iommu group %d\n", group->id); /* Pre-notify listeners that a device is being removed. */ blocking_notifier_call_chain(&group->notifier, @@ -1585,13 +1585,14 @@ static size_t iommu_pgsize(struct iommu_domain *domain, int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { + const struct iommu_ops *ops = domain->ops; unsigned long orig_iova = iova; unsigned int min_pagesz; size_t orig_size = size; phys_addr_t orig_paddr = paddr; int ret = 0; - if (unlikely(domain->ops->map == NULL || + if (unlikely(ops->map == NULL || domain->pgsize_bitmap == 0UL)) return -ENODEV; @@ -1620,7 +1621,7 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n", iova, &paddr, pgsize); - ret = domain->ops->map(domain, iova, paddr, pgsize, prot); + ret = ops->map(domain, iova, paddr, pgsize, prot); if (ret) break; @@ -1629,6 +1630,9 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, size -= pgsize; } + if (ops->iotlb_sync_map) + ops->iotlb_sync_map(domain); + /* unroll mapping in case something went wrong */ if (ret) iommu_unmap(domain, orig_iova, orig_size - size); @@ -1951,7 +1955,7 @@ int iommu_request_dm_for_dev(struct device *dev) iommu_domain_free(group->default_domain); group->default_domain = dm_domain; - pr_info("Using direct mapping for device %s\n", dev_name(dev)); + dev_info(dev, "Using iommu direct mapping\n"); ret = 0; out: diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 7a4529c61c19..9a380c10655e 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -15,6 +15,7 @@ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/io-pgtable.h> #include <linux/iommu.h> #include <linux/of.h> #include <linux/of_device.h> @@ -35,8 +36,6 @@ #define arm_iommu_detach_device(...) do {} while (0) #endif -#include "io-pgtable.h" - #define IPMMU_CTX_MAX 8 struct ipmmu_features { diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c index b94ebd42edd8..81cf2908c531 100644 --- a/drivers/iommu/irq_remapping.c +++ b/drivers/iommu/irq_remapping.c @@ -103,6 +103,9 @@ int __init irq_remapping_prepare(void) else if (IS_ENABLED(CONFIG_AMD_IOMMU) && amd_iommu_irq_ops.prepare() == 0) remap_ops = &amd_iommu_irq_ops; + else if (IS_ENABLED(CONFIG_HYPERV_IOMMU) && + hyperv_irq_remap_ops.prepare() == 0) + remap_ops = &hyperv_irq_remap_ops; else return -ENOSYS; diff --git a/drivers/iommu/irq_remapping.h b/drivers/iommu/irq_remapping.h index 0afef6e43be4..f8609e9f1f5d 100644 --- a/drivers/iommu/irq_remapping.h +++ b/drivers/iommu/irq_remapping.h @@ -64,6 +64,7 @@ struct irq_remap_ops { extern struct irq_remap_ops intel_irq_remap_ops; extern struct irq_remap_ops amd_iommu_irq_ops; +extern struct irq_remap_ops hyperv_irq_remap_ops; #else /* CONFIG_IRQ_REMAP */ diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index fc4270733f11..9fb0eb7a4d02 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -23,6 +23,7 @@ #include <linux/platform_device.h> #include <linux/errno.h> #include <linux/io.h> +#include <linux/io-pgtable.h> #include <linux/interrupt.h> #include <linux/list.h> #include <linux/spinlock.h> @@ -37,7 +38,6 @@ #include "msm_iommu_hw-8xxx.h" #include "msm_iommu.h" -#include "io-pgtable.h" #define MRC(reg, processor, op1, crn, crm, op2) \ __asm__ __volatile__ ( \ @@ -461,10 +461,10 @@ static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) master->num = msm_iommu_alloc_ctx(iommu->context_map, 0, iommu->ncb); - if (IS_ERR_VALUE(master->num)) { - ret = -ENODEV; - goto fail; - } + if (IS_ERR_VALUE(master->num)) { + ret = -ENODEV; + goto fail; + } config_mids(iommu, master); __program_context(iommu->base, master->num, priv); diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index 778498b8633f..62c2c3e8c5df 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -19,13 +19,12 @@ #include <linux/component.h> #include <linux/device.h> #include <linux/io.h> +#include <linux/io-pgtable.h> #include <linux/iommu.h> #include <linux/list.h> #include <linux/spinlock.h> #include <soc/mediatek/smi.h> -#include "io-pgtable.h" - struct mtk_iommu_suspend_reg { u32 standard_axi_mode; u32 dcm_dis; diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 6ede4286b835..52b01e3a49df 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -232,9 +232,8 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data) spin_lock_init(&dom->pgtlock); - dom->pgt_va = dma_zalloc_coherent(data->dev, - M2701_IOMMU_PGT_SIZE, - &dom->pgt_pa, GFP_KERNEL); + dom->pgt_va = dma_alloc_coherent(data->dev, M2701_IOMMU_PGT_SIZE, + &dom->pgt_pa, GFP_KERNEL); if (!dom->pgt_va) return -ENOMEM; @@ -442,6 +441,10 @@ static int mtk_iommu_add_device(struct device *dev) iommu_spec.args_count = count; mtk_iommu_create_mapping(dev, &iommu_spec); + + /* dev->iommu_fwspec might have changed */ + fwspec = dev_iommu_fwspec_get(dev); + of_node_put(iommu_spec.np); } @@ -471,7 +474,7 @@ static int mtk_iommu_add_device(struct device *dev) return err; } - return iommu_device_link(&data->iommu, dev);; + return iommu_device_link(&data->iommu, dev); } static void mtk_iommu_remove_device(struct device *dev) diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index d8947b28db2d..f04a6df65eb8 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -224,7 +224,7 @@ const struct iommu_ops *of_iommu_configure(struct device *dev, * If we have reason to believe the IOMMU driver missed the initial * probe for dev, replay it to get things in order. */ - if (dev->bus && !device_iommu_mapped(dev)) + if (!err && dev->bus && !device_iommu_mapped(dev)) err = iommu_probe_device(dev); /* Ignore all other errors apart from EPROBE_DEFER */ diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c index d8595f0a987d..8cdd3f059513 100644 --- a/drivers/iommu/qcom_iommu.c +++ b/drivers/iommu/qcom_iommu.c @@ -26,6 +26,7 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/io-64-nonatomic-hi-lo.h> +#include <linux/io-pgtable.h> #include <linux/iommu.h> #include <linux/iopoll.h> #include <linux/kconfig.h> @@ -42,7 +43,6 @@ #include <linux/slab.h> #include <linux/spinlock.h> -#include "io-pgtable.h" #include "arm-smmu-regs.h" #define SMMU_INTR_SEL_NS 0x2000 diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c index da6a4e357b2b..4d8057916552 100644 --- a/drivers/iommu/tegra-gart.c +++ b/drivers/iommu/tegra-gart.c @@ -1,5 +1,5 @@ /* - * IOMMU API for GART in Tegra20 + * IOMMU API for Graphics Address Relocation Table on Tegra20 * * Copyright (c) 2010-2012, NVIDIA CORPORATION. All rights reserved. * @@ -19,101 +19,75 @@ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. */ -#define pr_fmt(fmt) "%s(): " fmt, __func__ +#define dev_fmt(fmt) "gart: " fmt -#include <linux/init.h> +#include <linux/io.h> +#include <linux/iommu.h> #include <linux/moduleparam.h> #include <linux/platform_device.h> -#include <linux/spinlock.h> #include <linux/slab.h> +#include <linux/spinlock.h> #include <linux/vmalloc.h> -#include <linux/mm.h> -#include <linux/list.h> -#include <linux/device.h> -#include <linux/io.h> -#include <linux/iommu.h> -#include <linux/of.h> - -#include <asm/cacheflush.h> -/* bitmap of the page sizes currently supported */ -#define GART_IOMMU_PGSIZES (SZ_4K) +#include <soc/tegra/mc.h> #define GART_REG_BASE 0x24 #define GART_CONFIG (0x24 - GART_REG_BASE) #define GART_ENTRY_ADDR (0x28 - GART_REG_BASE) #define GART_ENTRY_DATA (0x2c - GART_REG_BASE) -#define GART_ENTRY_PHYS_ADDR_VALID (1 << 31) + +#define GART_ENTRY_PHYS_ADDR_VALID BIT(31) #define GART_PAGE_SHIFT 12 #define GART_PAGE_SIZE (1 << GART_PAGE_SHIFT) -#define GART_PAGE_MASK \ - (~(GART_PAGE_SIZE - 1) & ~GART_ENTRY_PHYS_ADDR_VALID) +#define GART_PAGE_MASK GENMASK(30, GART_PAGE_SHIFT) -struct gart_client { - struct device *dev; - struct list_head list; -}; +/* bitmap of the page sizes currently supported */ +#define GART_IOMMU_PGSIZES (GART_PAGE_SIZE) struct gart_device { void __iomem *regs; u32 *savedata; - u32 page_count; /* total remappable size */ - dma_addr_t iovmm_base; /* offset to vmm_area */ + unsigned long iovmm_base; /* offset to vmm_area start */ + unsigned long iovmm_end; /* offset to vmm_area end */ spinlock_t pte_lock; /* for pagetable */ - struct list_head client; - spinlock_t client_lock; /* for client list */ - struct device *dev; - + spinlock_t dom_lock; /* for active domain */ + unsigned int active_devices; /* number of active devices */ + struct iommu_domain *active_domain; /* current active domain */ struct iommu_device iommu; /* IOMMU Core handle */ -}; - -struct gart_domain { - struct iommu_domain domain; /* generic domain handle */ - struct gart_device *gart; /* link to gart device */ + struct device *dev; }; static struct gart_device *gart_handle; /* unique for a system */ static bool gart_debug; -#define GART_PTE(_pfn) \ - (GART_ENTRY_PHYS_ADDR_VALID | ((_pfn) << PAGE_SHIFT)) - -static struct gart_domain *to_gart_domain(struct iommu_domain *dom) -{ - return container_of(dom, struct gart_domain, domain); -} - /* * Any interaction between any block on PPSB and a block on APB or AHB * must have these read-back to ensure the APB/AHB bus transaction is * complete before initiating activity on the PPSB block. */ -#define FLUSH_GART_REGS(gart) ((void)readl((gart)->regs + GART_CONFIG)) +#define FLUSH_GART_REGS(gart) readl_relaxed((gart)->regs + GART_CONFIG) #define for_each_gart_pte(gart, iova) \ for (iova = gart->iovmm_base; \ - iova < gart->iovmm_base + GART_PAGE_SIZE * gart->page_count; \ + iova < gart->iovmm_end; \ iova += GART_PAGE_SIZE) static inline void gart_set_pte(struct gart_device *gart, - unsigned long offs, u32 pte) + unsigned long iova, unsigned long pte) { - writel(offs, gart->regs + GART_ENTRY_ADDR); - writel(pte, gart->regs + GART_ENTRY_DATA); - - dev_dbg(gart->dev, "%s %08lx:%08x\n", - pte ? "map" : "unmap", offs, pte & GART_PAGE_MASK); + writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR); + writel_relaxed(pte, gart->regs + GART_ENTRY_DATA); } static inline unsigned long gart_read_pte(struct gart_device *gart, - unsigned long offs) + unsigned long iova) { unsigned long pte; - writel(offs, gart->regs + GART_ENTRY_ADDR); - pte = readl(gart->regs + GART_ENTRY_DATA); + writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR); + pte = readl_relaxed(gart->regs + GART_ENTRY_DATA); return pte; } @@ -125,224 +99,155 @@ static void do_gart_setup(struct gart_device *gart, const u32 *data) for_each_gart_pte(gart, iova) gart_set_pte(gart, iova, data ? *(data++) : 0); - writel(1, gart->regs + GART_CONFIG); + writel_relaxed(1, gart->regs + GART_CONFIG); FLUSH_GART_REGS(gart); } -#ifdef DEBUG -static void gart_dump_table(struct gart_device *gart) -{ - unsigned long iova; - unsigned long flags; - - spin_lock_irqsave(&gart->pte_lock, flags); - for_each_gart_pte(gart, iova) { - unsigned long pte; - - pte = gart_read_pte(gart, iova); - - dev_dbg(gart->dev, "%s %08lx:%08lx\n", - (GART_ENTRY_PHYS_ADDR_VALID & pte) ? "v" : " ", - iova, pte & GART_PAGE_MASK); - } - spin_unlock_irqrestore(&gart->pte_lock, flags); -} -#else -static inline void gart_dump_table(struct gart_device *gart) +static inline bool gart_iova_range_invalid(struct gart_device *gart, + unsigned long iova, size_t bytes) { + return unlikely(iova < gart->iovmm_base || bytes != GART_PAGE_SIZE || + iova + bytes > gart->iovmm_end); } -#endif -static inline bool gart_iova_range_valid(struct gart_device *gart, - unsigned long iova, size_t bytes) +static inline bool gart_pte_valid(struct gart_device *gart, unsigned long iova) { - unsigned long iova_start, iova_end, gart_start, gart_end; - - iova_start = iova; - iova_end = iova_start + bytes - 1; - gart_start = gart->iovmm_base; - gart_end = gart_start + gart->page_count * GART_PAGE_SIZE - 1; - - if (iova_start < gart_start) - return false; - if (iova_end > gart_end) - return false; - return true; + return !!(gart_read_pte(gart, iova) & GART_ENTRY_PHYS_ADDR_VALID); } static int gart_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) { - struct gart_domain *gart_domain = to_gart_domain(domain); - struct gart_device *gart = gart_domain->gart; - struct gart_client *client, *c; - int err = 0; - - client = devm_kzalloc(gart->dev, sizeof(*c), GFP_KERNEL); - if (!client) - return -ENOMEM; - client->dev = dev; - - spin_lock(&gart->client_lock); - list_for_each_entry(c, &gart->client, list) { - if (c->dev == dev) { - dev_err(gart->dev, - "%s is already attached\n", dev_name(dev)); - err = -EINVAL; - goto fail; - } + struct gart_device *gart = gart_handle; + int ret = 0; + + spin_lock(&gart->dom_lock); + + if (gart->active_domain && gart->active_domain != domain) { + ret = -EBUSY; + } else if (dev->archdata.iommu != domain) { + dev->archdata.iommu = domain; + gart->active_domain = domain; + gart->active_devices++; } - list_add(&client->list, &gart->client); - spin_unlock(&gart->client_lock); - dev_dbg(gart->dev, "Attached %s\n", dev_name(dev)); - return 0; -fail: - devm_kfree(gart->dev, client); - spin_unlock(&gart->client_lock); - return err; + spin_unlock(&gart->dom_lock); + + return ret; } static void gart_iommu_detach_dev(struct iommu_domain *domain, struct device *dev) { - struct gart_domain *gart_domain = to_gart_domain(domain); - struct gart_device *gart = gart_domain->gart; - struct gart_client *c; - - spin_lock(&gart->client_lock); - - list_for_each_entry(c, &gart->client, list) { - if (c->dev == dev) { - list_del(&c->list); - devm_kfree(gart->dev, c); - dev_dbg(gart->dev, "Detached %s\n", dev_name(dev)); - goto out; - } + struct gart_device *gart = gart_handle; + + spin_lock(&gart->dom_lock); + + if (dev->archdata.iommu == domain) { + dev->archdata.iommu = NULL; + + if (--gart->active_devices == 0) + gart->active_domain = NULL; } - dev_err(gart->dev, "Couldn't find\n"); -out: - spin_unlock(&gart->client_lock); + + spin_unlock(&gart->dom_lock); } static struct iommu_domain *gart_iommu_domain_alloc(unsigned type) { - struct gart_domain *gart_domain; - struct gart_device *gart; + struct iommu_domain *domain; if (type != IOMMU_DOMAIN_UNMANAGED) return NULL; - gart = gart_handle; - if (!gart) - return NULL; - - gart_domain = kzalloc(sizeof(*gart_domain), GFP_KERNEL); - if (!gart_domain) - return NULL; - - gart_domain->gart = gart; - gart_domain->domain.geometry.aperture_start = gart->iovmm_base; - gart_domain->domain.geometry.aperture_end = gart->iovmm_base + - gart->page_count * GART_PAGE_SIZE - 1; - gart_domain->domain.geometry.force_aperture = true; + domain = kzalloc(sizeof(*domain), GFP_KERNEL); + if (domain) { + domain->geometry.aperture_start = gart_handle->iovmm_base; + domain->geometry.aperture_end = gart_handle->iovmm_end - 1; + domain->geometry.force_aperture = true; + } - return &gart_domain->domain; + return domain; } static void gart_iommu_domain_free(struct iommu_domain *domain) { - struct gart_domain *gart_domain = to_gart_domain(domain); - struct gart_device *gart = gart_domain->gart; - - if (gart) { - spin_lock(&gart->client_lock); - if (!list_empty(&gart->client)) { - struct gart_client *c; - - list_for_each_entry(c, &gart->client, list) - gart_iommu_detach_dev(domain, c->dev); - } - spin_unlock(&gart->client_lock); + WARN_ON(gart_handle->active_domain == domain); + kfree(domain); +} + +static inline int __gart_iommu_map(struct gart_device *gart, unsigned long iova, + unsigned long pa) +{ + if (unlikely(gart_debug && gart_pte_valid(gart, iova))) { + dev_err(gart->dev, "Page entry is in-use\n"); + return -EINVAL; } - kfree(gart_domain); + gart_set_pte(gart, iova, GART_ENTRY_PHYS_ADDR_VALID | pa); + + return 0; } static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t pa, size_t bytes, int prot) { - struct gart_domain *gart_domain = to_gart_domain(domain); - struct gart_device *gart = gart_domain->gart; - unsigned long flags; - unsigned long pfn; - unsigned long pte; + struct gart_device *gart = gart_handle; + int ret; - if (!gart_iova_range_valid(gart, iova, bytes)) + if (gart_iova_range_invalid(gart, iova, bytes)) return -EINVAL; - spin_lock_irqsave(&gart->pte_lock, flags); - pfn = __phys_to_pfn(pa); - if (!pfn_valid(pfn)) { - dev_err(gart->dev, "Invalid page: %pa\n", &pa); - spin_unlock_irqrestore(&gart->pte_lock, flags); + spin_lock(&gart->pte_lock); + ret = __gart_iommu_map(gart, iova, (unsigned long)pa); + spin_unlock(&gart->pte_lock); + + return ret; +} + +static inline int __gart_iommu_unmap(struct gart_device *gart, + unsigned long iova) +{ + if (unlikely(gart_debug && !gart_pte_valid(gart, iova))) { + dev_err(gart->dev, "Page entry is invalid\n"); return -EINVAL; } - if (gart_debug) { - pte = gart_read_pte(gart, iova); - if (pte & GART_ENTRY_PHYS_ADDR_VALID) { - spin_unlock_irqrestore(&gart->pte_lock, flags); - dev_err(gart->dev, "Page entry is in-use\n"); - return -EBUSY; - } - } - gart_set_pte(gart, iova, GART_PTE(pfn)); - FLUSH_GART_REGS(gart); - spin_unlock_irqrestore(&gart->pte_lock, flags); + + gart_set_pte(gart, iova, 0); + return 0; } static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t bytes) { - struct gart_domain *gart_domain = to_gart_domain(domain); - struct gart_device *gart = gart_domain->gart; - unsigned long flags; + struct gart_device *gart = gart_handle; + int err; - if (!gart_iova_range_valid(gart, iova, bytes)) + if (gart_iova_range_invalid(gart, iova, bytes)) return 0; - spin_lock_irqsave(&gart->pte_lock, flags); - gart_set_pte(gart, iova, 0); - FLUSH_GART_REGS(gart); - spin_unlock_irqrestore(&gart->pte_lock, flags); - return bytes; + spin_lock(&gart->pte_lock); + err = __gart_iommu_unmap(gart, iova); + spin_unlock(&gart->pte_lock); + + return err ? 0 : bytes; } static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { - struct gart_domain *gart_domain = to_gart_domain(domain); - struct gart_device *gart = gart_domain->gart; + struct gart_device *gart = gart_handle; unsigned long pte; - phys_addr_t pa; - unsigned long flags; - if (!gart_iova_range_valid(gart, iova, 0)) + if (gart_iova_range_invalid(gart, iova, GART_PAGE_SIZE)) return -EINVAL; - spin_lock_irqsave(&gart->pte_lock, flags); + spin_lock(&gart->pte_lock); pte = gart_read_pte(gart, iova); - spin_unlock_irqrestore(&gart->pte_lock, flags); + spin_unlock(&gart->pte_lock); - pa = (pte & GART_PAGE_MASK); - if (!pfn_valid(__phys_to_pfn(pa))) { - dev_err(gart->dev, "No entry for %08llx:%pa\n", - (unsigned long long)iova, &pa); - gart_dump_table(gart); - return -EINVAL; - } - return pa; + return pte & GART_PAGE_MASK; } static bool gart_iommu_capable(enum iommu_cap cap) @@ -352,8 +257,12 @@ static bool gart_iommu_capable(enum iommu_cap cap) static int gart_iommu_add_device(struct device *dev) { - struct iommu_group *group = iommu_group_get_for_dev(dev); + struct iommu_group *group; + + if (!dev->iommu_fwspec) + return -ENODEV; + group = iommu_group_get_for_dev(dev); if (IS_ERR(group)) return PTR_ERR(group); @@ -370,6 +279,17 @@ static void gart_iommu_remove_device(struct device *dev) iommu_device_unlink(&gart_handle->iommu, dev); } +static int gart_iommu_of_xlate(struct device *dev, + struct of_phandle_args *args) +{ + return 0; +} + +static void gart_iommu_sync(struct iommu_domain *domain) +{ + FLUSH_GART_REGS(gart_handle); +} + static const struct iommu_ops gart_iommu_ops = { .capable = gart_iommu_capable, .domain_alloc = gart_iommu_domain_alloc, @@ -383,129 +303,96 @@ static const struct iommu_ops gart_iommu_ops = { .unmap = gart_iommu_unmap, .iova_to_phys = gart_iommu_iova_to_phys, .pgsize_bitmap = GART_IOMMU_PGSIZES, + .of_xlate = gart_iommu_of_xlate, + .iotlb_sync_map = gart_iommu_sync, + .iotlb_sync = gart_iommu_sync, }; -static int tegra_gart_suspend(struct device *dev) +int tegra_gart_suspend(struct gart_device *gart) { - struct gart_device *gart = dev_get_drvdata(dev); - unsigned long iova; u32 *data = gart->savedata; - unsigned long flags; + unsigned long iova; + + /* + * All GART users shall be suspended at this point. Disable + * address translation to trap all GART accesses as invalid + * memory accesses. + */ + writel_relaxed(0, gart->regs + GART_CONFIG); + FLUSH_GART_REGS(gart); - spin_lock_irqsave(&gart->pte_lock, flags); for_each_gart_pte(gart, iova) *(data++) = gart_read_pte(gart, iova); - spin_unlock_irqrestore(&gart->pte_lock, flags); + return 0; } -static int tegra_gart_resume(struct device *dev) +int tegra_gart_resume(struct gart_device *gart) { - struct gart_device *gart = dev_get_drvdata(dev); - unsigned long flags; - - spin_lock_irqsave(&gart->pte_lock, flags); do_gart_setup(gart, gart->savedata); - spin_unlock_irqrestore(&gart->pte_lock, flags); + return 0; } -static int tegra_gart_probe(struct platform_device *pdev) +struct gart_device *tegra_gart_probe(struct device *dev, struct tegra_mc *mc) { struct gart_device *gart; - struct resource *res, *res_remap; - void __iomem *gart_regs; - struct device *dev = &pdev->dev; - int ret; - - if (gart_handle) - return -EIO; + struct resource *res; + int err; BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT); /* the GART memory aperture is required */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - res_remap = platform_get_resource(pdev, IORESOURCE_MEM, 1); - if (!res || !res_remap) { - dev_err(dev, "GART memory aperture expected\n"); - return -ENXIO; - } - - gart = devm_kzalloc(dev, sizeof(*gart), GFP_KERNEL); - if (!gart) { - dev_err(dev, "failed to allocate gart_device\n"); - return -ENOMEM; + res = platform_get_resource(to_platform_device(dev), IORESOURCE_MEM, 1); + if (!res) { + dev_err(dev, "Memory aperture resource unavailable\n"); + return ERR_PTR(-ENXIO); } - gart_regs = devm_ioremap(dev, res->start, resource_size(res)); - if (!gart_regs) { - dev_err(dev, "failed to remap GART registers\n"); - return -ENXIO; - } - - ret = iommu_device_sysfs_add(&gart->iommu, &pdev->dev, NULL, - dev_name(&pdev->dev)); - if (ret) { - dev_err(dev, "Failed to register IOMMU in sysfs\n"); - return ret; - } - - iommu_device_set_ops(&gart->iommu, &gart_iommu_ops); + gart = kzalloc(sizeof(*gart), GFP_KERNEL); + if (!gart) + return ERR_PTR(-ENOMEM); - ret = iommu_device_register(&gart->iommu); - if (ret) { - dev_err(dev, "Failed to register IOMMU\n"); - iommu_device_sysfs_remove(&gart->iommu); - return ret; - } + gart_handle = gart; - gart->dev = &pdev->dev; + gart->dev = dev; + gart->regs = mc->regs + GART_REG_BASE; + gart->iovmm_base = res->start; + gart->iovmm_end = res->end + 1; spin_lock_init(&gart->pte_lock); - spin_lock_init(&gart->client_lock); - INIT_LIST_HEAD(&gart->client); - gart->regs = gart_regs; - gart->iovmm_base = (dma_addr_t)res_remap->start; - gart->page_count = (resource_size(res_remap) >> GART_PAGE_SHIFT); - - gart->savedata = vmalloc(array_size(sizeof(u32), gart->page_count)); - if (!gart->savedata) { - dev_err(dev, "failed to allocate context save area\n"); - return -ENOMEM; - } + spin_lock_init(&gart->dom_lock); - platform_set_drvdata(pdev, gart); do_gart_setup(gart, NULL); - gart_handle = gart; + err = iommu_device_sysfs_add(&gart->iommu, dev, NULL, "gart"); + if (err) + goto free_gart; - return 0; -} + iommu_device_set_ops(&gart->iommu, &gart_iommu_ops); + iommu_device_set_fwnode(&gart->iommu, dev->fwnode); -static const struct dev_pm_ops tegra_gart_pm_ops = { - .suspend = tegra_gart_suspend, - .resume = tegra_gart_resume, -}; + err = iommu_device_register(&gart->iommu); + if (err) + goto remove_sysfs; -static const struct of_device_id tegra_gart_of_match[] = { - { .compatible = "nvidia,tegra20-gart", }, - { }, -}; + gart->savedata = vmalloc(resource_size(res) / GART_PAGE_SIZE * + sizeof(u32)); + if (!gart->savedata) { + err = -ENOMEM; + goto unregister_iommu; + } -static struct platform_driver tegra_gart_driver = { - .probe = tegra_gart_probe, - .driver = { - .name = "tegra-gart", - .pm = &tegra_gart_pm_ops, - .of_match_table = tegra_gart_of_match, - .suppress_bind_attrs = true, - }, -}; + return gart; -static int __init tegra_gart_init(void) -{ - return platform_driver_register(&tegra_gart_driver); +unregister_iommu: + iommu_device_unregister(&gart->iommu); +remove_sysfs: + iommu_device_sysfs_remove(&gart->iommu); +free_gart: + kfree(gart); + + return ERR_PTR(err); } -subsys_initcall(tegra_gart_init); module_param(gart_debug, bool, 0644); MODULE_PARM_DESC(gart_debug, "Enable GART debugging"); diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 3a5c7dc6dc57..5182c7d6171e 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -982,10 +982,6 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev, u32 value; int err; - /* This can happen on Tegra20 which doesn't have an SMMU */ - if (!soc) - return NULL; - smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); if (!smmu) return ERR_PTR(-ENOMEM); |