diff options
Diffstat (limited to 'drivers/iommu')
-rw-r--r-- | drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 | ||||
-rw-r--r-- | drivers/iommu/dma-iommu.c | 4 | ||||
-rw-r--r-- | drivers/iommu/exynos-iommu.c | 17 | ||||
-rw-r--r-- | drivers/iommu/intel/dmar.c | 3 | ||||
-rw-r--r-- | drivers/iommu/intel/iommu.h | 2 | ||||
-rw-r--r-- | drivers/iommu/intel/irq_remapping.c | 6 | ||||
-rw-r--r-- | drivers/iommu/intel/perfmon.c | 68 | ||||
-rw-r--r-- | drivers/iommu/iommu.c | 18 | ||||
-rw-r--r-- | drivers/iommu/iommufd/device.c | 205 | ||||
-rw-r--r-- | drivers/iommu/iommufd/hw_pagetable.c | 70 | ||||
-rw-r--r-- | drivers/iommu/iommufd/ioas.c | 14 | ||||
-rw-r--r-- | drivers/iommu/iommufd/iommufd_private.h | 39 | ||||
-rw-r--r-- | drivers/iommu/iommufd/iommufd_test.h | 2 | ||||
-rw-r--r-- | drivers/iommu/iommufd/pages.c | 16 | ||||
-rw-r--r-- | drivers/iommu/iommufd/selftest.c | 219 | ||||
-rw-r--r-- | drivers/iommu/iommufd/vfio_compat.c | 2 | ||||
-rw-r--r-- | drivers/iommu/sun50i-iommu.c | 1 |
17 files changed, 450 insertions, 238 deletions
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 8d772ea8a583..b574c58a3487 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -182,7 +182,7 @@ #ifdef CONFIG_CMA_ALIGNMENT #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT) #else -#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1) +#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER) #endif /* diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 99b2646cb5c7..7a9f0b0bddbd 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -736,7 +736,7 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev, struct page **pages; unsigned int i = 0, nid = dev_to_node(dev); - order_mask &= (2U << MAX_ORDER) - 1; + order_mask &= GENMASK(MAX_ORDER, 0); if (!order_mask) return NULL; @@ -756,7 +756,7 @@ static struct page **__iommu_dma_alloc_pages(struct device *dev, * than a necessity, hence using __GFP_NORETRY until * falling back to minimum-order allocations. */ - for (order_mask &= (2U << __fls(count)) - 1; + for (order_mask &= GENMASK(__fls(count), 0); order_mask; order_mask &= ~order_size) { unsigned int order = __fls(order_mask); gfp_t alloc_flags = gfp; diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 483aaaeb6dae..1abd187c6075 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1415,23 +1415,26 @@ static struct iommu_device *exynos_iommu_probe_device(struct device *dev) return &data->iommu; } -static void exynos_iommu_release_device(struct device *dev) +static void exynos_iommu_set_platform_dma(struct device *dev) { struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); - struct sysmmu_drvdata *data; if (owner->domain) { struct iommu_group *group = iommu_group_get(dev); if (group) { -#ifndef CONFIG_ARM - WARN_ON(owner->domain != - iommu_group_default_domain(group)); -#endif exynos_iommu_detach_device(owner->domain, dev); iommu_group_put(group); } } +} + +static void exynos_iommu_release_device(struct device *dev) +{ + struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); + struct sysmmu_drvdata *data; + + exynos_iommu_set_platform_dma(dev); list_for_each_entry(data, &owner->controllers, owner_node) device_link_del(data->link); @@ -1479,7 +1482,7 @@ static const struct iommu_ops exynos_iommu_ops = { .domain_alloc = exynos_iommu_domain_alloc, .device_group = generic_device_group, #ifdef CONFIG_ARM - .set_platform_dma_ops = exynos_iommu_release_device, + .set_platform_dma_ops = exynos_iommu_set_platform_dma, #endif .probe_device = exynos_iommu_probe_device, .release_device = exynos_iommu_release_device, diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 6acfe879589c..23828d189c2a 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -1071,7 +1071,8 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) } err = -EINVAL; - if (cap_sagaw(iommu->cap) == 0) { + if (!cap_sagaw(iommu->cap) && + (!ecap_smts(iommu->ecap) || ecap_slts(iommu->ecap))) { pr_info("%s: No supported address widths. Not attempting DMA translation.\n", iommu->name); drhd->ignored = 1; diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index d6df3b865812..694ab9b7d3e9 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -641,6 +641,8 @@ struct iommu_pmu { DECLARE_BITMAP(used_mask, IOMMU_PMU_IDX_MAX); struct perf_event *event_list[IOMMU_PMU_IDX_MAX]; unsigned char irq_name[16]; + struct hlist_node cpuhp_node; + int cpu; }; #define IOMMU_IRQ_ID_OFFSET_PRQ (DMAR_UNITS_SUPPORTED) diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index 6d01fa078c36..df9e261af0b5 100644 --- a/drivers/iommu/intel/irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -311,14 +311,12 @@ static int set_ioapic_sid(struct irte *irte, int apic) if (!irte) return -1; - down_read(&dmar_global_lock); for (i = 0; i < MAX_IO_APICS; i++) { if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) { sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; break; } } - up_read(&dmar_global_lock); if (sid == 0) { pr_warn("Failed to set source-id of IOAPIC (%d)\n", apic); @@ -338,14 +336,12 @@ static int set_hpet_sid(struct irte *irte, u8 id) if (!irte) return -1; - down_read(&dmar_global_lock); for (i = 0; i < MAX_HPET_TBS; i++) { if (ir_hpet[i].iommu && ir_hpet[i].id == id) { sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; break; } } - up_read(&dmar_global_lock); if (sid == 0) { pr_warn("Failed to set source-id of HPET block (%d)\n", id); @@ -1339,9 +1335,7 @@ static int intel_irq_remapping_alloc(struct irq_domain *domain, if (!data) goto out_free_parent; - down_read(&dmar_global_lock); index = alloc_irte(iommu, &data->irq_2_iommu, nr_irqs); - up_read(&dmar_global_lock); if (index < 0) { pr_warn("Failed to allocate IRTE\n"); kfree(data); diff --git a/drivers/iommu/intel/perfmon.c b/drivers/iommu/intel/perfmon.c index e17d9743a0d8..cf43e798eca4 100644 --- a/drivers/iommu/intel/perfmon.c +++ b/drivers/iommu/intel/perfmon.c @@ -773,19 +773,34 @@ static void iommu_pmu_unset_interrupt(struct intel_iommu *iommu) iommu->perf_irq = 0; } -static int iommu_pmu_cpu_online(unsigned int cpu) +static int iommu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) { + struct iommu_pmu *iommu_pmu = hlist_entry_safe(node, typeof(*iommu_pmu), cpuhp_node); + if (cpumask_empty(&iommu_pmu_cpu_mask)) cpumask_set_cpu(cpu, &iommu_pmu_cpu_mask); + if (cpumask_test_cpu(cpu, &iommu_pmu_cpu_mask)) + iommu_pmu->cpu = cpu; + return 0; } -static int iommu_pmu_cpu_offline(unsigned int cpu) +static int iommu_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node) { - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; - int target; + struct iommu_pmu *iommu_pmu = hlist_entry_safe(node, typeof(*iommu_pmu), cpuhp_node); + int target = cpumask_first(&iommu_pmu_cpu_mask); + + /* + * The iommu_pmu_cpu_mask has been updated when offline the CPU + * for the first iommu_pmu. Migrate the other iommu_pmu to the + * new target. + */ + if (target < nr_cpu_ids && target != iommu_pmu->cpu) { + perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target); + iommu_pmu->cpu = target; + return 0; + } if (!cpumask_test_and_clear_cpu(cpu, &iommu_pmu_cpu_mask)) return 0; @@ -795,45 +810,50 @@ static int iommu_pmu_cpu_offline(unsigned int cpu) if (target < nr_cpu_ids) cpumask_set_cpu(target, &iommu_pmu_cpu_mask); else - target = -1; + return 0; - rcu_read_lock(); - - for_each_iommu(iommu, drhd) { - if (!iommu->pmu) - continue; - perf_pmu_migrate_context(&iommu->pmu->pmu, cpu, target); - } - rcu_read_unlock(); + perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target); + iommu_pmu->cpu = target; return 0; } static int nr_iommu_pmu; +static enum cpuhp_state iommu_cpuhp_slot; static int iommu_pmu_cpuhp_setup(struct iommu_pmu *iommu_pmu) { int ret; - if (nr_iommu_pmu++) - return 0; + if (!nr_iommu_pmu) { + ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, + "driver/iommu/intel/perfmon:online", + iommu_pmu_cpu_online, + iommu_pmu_cpu_offline); + if (ret < 0) + return ret; + iommu_cpuhp_slot = ret; + } - ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_IOMMU_PERF_ONLINE, - "driver/iommu/intel/perfmon:online", - iommu_pmu_cpu_online, - iommu_pmu_cpu_offline); - if (ret) - nr_iommu_pmu = 0; + ret = cpuhp_state_add_instance(iommu_cpuhp_slot, &iommu_pmu->cpuhp_node); + if (ret) { + if (!nr_iommu_pmu) + cpuhp_remove_multi_state(iommu_cpuhp_slot); + return ret; + } + nr_iommu_pmu++; - return ret; + return 0; } static void iommu_pmu_cpuhp_free(struct iommu_pmu *iommu_pmu) { + cpuhp_state_remove_instance(iommu_cpuhp_slot, &iommu_pmu->cpuhp_node); + if (--nr_iommu_pmu) return; - cpuhp_remove_state(CPUHP_AP_PERF_X86_IOMMU_PERF_ONLINE); + cpuhp_remove_multi_state(iommu_cpuhp_slot); } void iommu_pmu_register(struct intel_iommu *iommu) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 10db680acaed..807c98de40d4 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -28,6 +28,7 @@ #include <linux/fsl/mc.h> #include <linux/module.h> #include <linux/cc_platform.h> +#include <linux/cdx/cdx_bus.h> #include <trace/events/iommu.h> #include <linux/sched/mm.h> #include <linux/msi.h> @@ -89,7 +90,7 @@ static int iommu_bus_notifier(struct notifier_block *nb, unsigned long action, void *data); static int iommu_alloc_default_domain(struct iommu_group *group, struct device *dev); -static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, +static struct iommu_domain *__iommu_domain_alloc(const struct bus_type *bus, unsigned type); static int __iommu_attach_device(struct iommu_domain *domain, struct device *dev); @@ -129,6 +130,9 @@ static struct bus_type * const iommu_buses[] = { #ifdef CONFIG_TEGRA_HOST1X_CONTEXT_BUS &host1x_context_device_bus_type, #endif +#ifdef CONFIG_CDX_BUS + &cdx_bus_type, +#endif }; /* @@ -1631,7 +1635,7 @@ static int iommu_get_def_domain_type(struct device *dev) return 0; } -static int iommu_group_alloc_default_domain(struct bus_type *bus, +static int iommu_group_alloc_default_domain(const struct bus_type *bus, struct iommu_group *group, unsigned int type) { @@ -1777,7 +1781,7 @@ static int probe_get_default_domain_type(struct device *dev, void *data) return 0; } -static void probe_alloc_default_domain(struct bus_type *bus, +static void probe_alloc_default_domain(const struct bus_type *bus, struct iommu_group *group) { struct __group_domain_type gtype; @@ -1832,7 +1836,7 @@ static int iommu_group_create_direct_mappings(struct iommu_group *group) iommu_do_create_direct_mappings); } -int bus_iommu_probe(struct bus_type *bus) +int bus_iommu_probe(const struct bus_type *bus) { struct iommu_group *group, *next; LIST_HEAD(group_list); @@ -1876,7 +1880,7 @@ int bus_iommu_probe(struct bus_type *bus) return ret; } -bool iommu_present(struct bus_type *bus) +bool iommu_present(const struct bus_type *bus) { return bus->iommu_ops != NULL; } @@ -1951,7 +1955,7 @@ void iommu_set_fault_handler(struct iommu_domain *domain, } EXPORT_SYMBOL_GPL(iommu_set_fault_handler); -static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, +static struct iommu_domain *__iommu_domain_alloc(const struct bus_type *bus, unsigned type) { struct iommu_domain *domain; @@ -1976,7 +1980,7 @@ static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, return domain; } -struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) +struct iommu_domain *iommu_domain_alloc(const struct bus_type *bus) { return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED); } diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c index a0c66f47a65a..4f9b2142274c 100644 --- a/drivers/iommu/iommufd/device.c +++ b/drivers/iommu/iommufd/device.c @@ -15,23 +15,6 @@ MODULE_PARM_DESC( "Allow IOMMUFD to bind to devices even if the platform cannot isolate " "the MSI interrupt window. Enabling this is a security weakness."); -/* - * A iommufd_device object represents the binding relationship between a - * consuming driver and the iommufd. These objects are created/destroyed by - * external drivers, not by userspace. - */ -struct iommufd_device { - struct iommufd_object obj; - struct iommufd_ctx *ictx; - struct iommufd_hw_pagetable *hwpt; - /* Head at iommufd_hw_pagetable::devices */ - struct list_head devices_item; - /* always the physical device */ - struct device *dev; - struct iommu_group *group; - bool enforce_cache_coherency; -}; - void iommufd_device_destroy(struct iommufd_object *obj) { struct iommufd_device *idev = @@ -39,7 +22,8 @@ void iommufd_device_destroy(struct iommufd_object *obj) iommu_device_release_dma_owner(idev->dev); iommu_group_put(idev->group); - iommufd_ctx_put(idev->ictx); + if (!iommufd_selftest_is_mock_dev(idev->dev)) + iommufd_ctx_put(idev->ictx); } /** @@ -86,7 +70,8 @@ struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx, goto out_release_owner; } idev->ictx = ictx; - iommufd_ctx_get(ictx); + if (!iommufd_selftest_is_mock_dev(dev)) + iommufd_ctx_get(ictx); idev->dev = dev; idev->enforce_cache_coherency = device_iommu_capable(dev, IOMMU_CAP_ENFORCE_CACHE_COHERENCY); @@ -168,7 +153,8 @@ static int iommufd_device_setup_msi(struct iommufd_device *idev, * operation from the device (eg a simple DMA) cannot trigger an * interrupt outside this iommufd context. */ - if (!iommu_group_has_isolated_msi(idev->group)) { + if (!iommufd_selftest_is_mock_dev(idev->dev) && + !iommu_group_has_isolated_msi(idev->group)) { if (!allow_unsafe_interrupts) return -EPERM; @@ -186,19 +172,24 @@ static bool iommufd_hw_pagetable_has_group(struct iommufd_hw_pagetable *hwpt, { struct iommufd_device *cur_dev; + lockdep_assert_held(&hwpt->devices_lock); + list_for_each_entry(cur_dev, &hwpt->devices, devices_item) if (cur_dev->group == group) return true; return false; } -static int iommufd_device_do_attach(struct iommufd_device *idev, - struct iommufd_hw_pagetable *hwpt) +int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt, + struct iommufd_device *idev) { phys_addr_t sw_msi_start = PHYS_ADDR_MAX; int rc; - mutex_lock(&hwpt->devices_lock); + lockdep_assert_held(&hwpt->devices_lock); + + if (WARN_ON(idev->hwpt)) + return -EINVAL; /* * Try to upgrade the domain we have, it is an iommu driver bug to @@ -213,19 +204,18 @@ static int iommufd_device_do_attach(struct iommufd_device *idev, hwpt->domain); if (!hwpt->enforce_cache_coherency) { WARN_ON(list_empty(&hwpt->devices)); - rc = -EINVAL; - goto out_unlock; + return -EINVAL; } } rc = iopt_table_enforce_group_resv_regions(&hwpt->ioas->iopt, idev->dev, idev->group, &sw_msi_start); if (rc) - goto out_unlock; + return rc; rc = iommufd_device_setup_msi(idev, hwpt, sw_msi_start); if (rc) - goto out_iova; + goto err_unresv; /* * FIXME: Hack around missing a device-centric iommu api, only attach to @@ -234,26 +224,35 @@ static int iommufd_device_do_attach(struct iommufd_device *idev, if (!iommufd_hw_pagetable_has_group(hwpt, idev->group)) { rc = iommu_attach_group(hwpt->domain, idev->group); if (rc) - goto out_iova; - - if (list_empty(&hwpt->devices)) { - rc = iopt_table_add_domain(&hwpt->ioas->iopt, - hwpt->domain); - if (rc) - goto out_detach; - } + goto err_unresv; } + return 0; +err_unresv: + iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev); + return rc; +} + +void iommufd_hw_pagetable_detach(struct iommufd_hw_pagetable *hwpt, + struct iommufd_device *idev) +{ + if (!iommufd_hw_pagetable_has_group(hwpt, idev->group)) + iommu_detach_group(hwpt->domain, idev->group); + iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev); +} + +static int iommufd_device_do_attach(struct iommufd_device *idev, + struct iommufd_hw_pagetable *hwpt) +{ + int rc; + + mutex_lock(&hwpt->devices_lock); + rc = iommufd_hw_pagetable_attach(hwpt, idev); + if (rc) + goto out_unlock; idev->hwpt = hwpt; refcount_inc(&hwpt->obj.users); list_add(&idev->devices_item, &hwpt->devices); - mutex_unlock(&hwpt->devices_lock); - return 0; - -out_detach: - iommu_detach_group(hwpt->domain, idev->group); -out_iova: - iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev); out_unlock: mutex_unlock(&hwpt->devices_lock); return rc; @@ -280,7 +279,10 @@ static int iommufd_device_auto_get_domain(struct iommufd_device *idev, if (!hwpt->auto_domain) continue; + if (!iommufd_lock_obj(&hwpt->obj)) + continue; rc = iommufd_device_do_attach(idev, hwpt); + iommufd_put_object(&hwpt->obj); /* * -EINVAL means the domain is incompatible with the device. @@ -292,24 +294,16 @@ static int iommufd_device_auto_get_domain(struct iommufd_device *idev, goto out_unlock; } - hwpt = iommufd_hw_pagetable_alloc(idev->ictx, ioas, idev->dev); + hwpt = iommufd_hw_pagetable_alloc(idev->ictx, ioas, idev, true); if (IS_ERR(hwpt)) { rc = PTR_ERR(hwpt); goto out_unlock; } hwpt->auto_domain = true; - rc = iommufd_device_do_attach(idev, hwpt); - if (rc) - goto out_abort; - list_add_tail(&hwpt->hwpt_item, &ioas->hwpt_list); - mutex_unlock(&ioas->mutex); iommufd_object_finalize(idev->ictx, &hwpt->obj); return 0; - -out_abort: - iommufd_object_abort_and_destroy(idev->ictx, &hwpt->obj); out_unlock: mutex_unlock(&ioas->mutex); return rc; @@ -381,28 +375,17 @@ void iommufd_device_detach(struct iommufd_device *idev) { struct iommufd_hw_pagetable *hwpt = idev->hwpt; - mutex_lock(&hwpt->ioas->mutex); mutex_lock(&hwpt->devices_lock); list_del(&idev->devices_item); - if (!iommufd_hw_pagetable_has_group(hwpt, idev->group)) { - if (list_empty(&hwpt->devices)) { - iopt_table_remove_domain(&hwpt->ioas->iopt, - hwpt->domain); - list_del(&hwpt->hwpt_item); - } - iommu_detach_group(hwpt->domain, idev->group); - } - iopt_remove_reserved_iova(&hwpt->ioas->iopt, idev->dev); + idev->hwpt = NULL; + iommufd_hw_pagetable_detach(hwpt, idev); mutex_unlock(&hwpt->devices_lock); - mutex_unlock(&hwpt->ioas->mutex); if (hwpt->auto_domain) iommufd_object_destroy_user(idev->ictx, &hwpt->obj); else refcount_dec(&hwpt->obj.users); - idev->hwpt = NULL; - refcount_dec(&idev->obj.users); } EXPORT_SYMBOL_NS_GPL(iommufd_device_detach, IOMMUFD); @@ -412,17 +395,20 @@ void iommufd_access_destroy_object(struct iommufd_object *obj) struct iommufd_access *access = container_of(obj, struct iommufd_access, obj); - iopt_remove_access(&access->ioas->iopt, access); + if (access->ioas) { + iopt_remove_access(&access->ioas->iopt, access); + refcount_dec(&access->ioas->obj.users); + access->ioas = NULL; + } iommufd_ctx_put(access->ictx); - refcount_dec(&access->ioas->obj.users); } /** * iommufd_access_create - Create an iommufd_access * @ictx: iommufd file descriptor - * @ioas_id: ID for a IOMMUFD_OBJ_IOAS * @ops: Driver's ops to associate with the access * @data: Opaque data to pass into ops functions + * @id: Output ID number to return to userspace for this access * * An iommufd_access allows a driver to read/write to the IOAS without using * DMA. The underlying CPU memory can be accessed using the @@ -431,12 +417,10 @@ void iommufd_access_destroy_object(struct iommufd_object *obj) * The provided ops are required to use iommufd_access_pin_pages(). */ struct iommufd_access * -iommufd_access_create(struct iommufd_ctx *ictx, u32 ioas_id, - const struct iommufd_access_ops *ops, void *data) +iommufd_access_create(struct iommufd_ctx *ictx, + const struct iommufd_access_ops *ops, void *data, u32 *id) { struct iommufd_access *access; - struct iommufd_object *obj; - int rc; /* * There is no uAPI for the access object, but to keep things symmetric @@ -449,33 +433,18 @@ iommufd_access_create(struct iommufd_ctx *ictx, u32 ioas_id, access->data = data; access->ops = ops; - obj = iommufd_get_object(ictx, ioas_id, IOMMUFD_OBJ_IOAS); - if (IS_ERR(obj)) { - rc = PTR_ERR(obj); - goto out_abort; - } - access->ioas = container_of(obj, struct iommufd_ioas, obj); - iommufd_ref_to_users(obj); - if (ops->needs_pin_pages) access->iova_alignment = PAGE_SIZE; else access->iova_alignment = 1; - rc = iopt_add_access(&access->ioas->iopt, access); - if (rc) - goto out_put_ioas; /* The calling driver is a user until iommufd_access_destroy() */ refcount_inc(&access->obj.users); access->ictx = ictx; iommufd_ctx_get(ictx); iommufd_object_finalize(ictx, &access->obj); + *id = access->obj.id; return access; -out_put_ioas: - refcount_dec(&access->ioas->obj.users); -out_abort: - iommufd_object_abort(ictx, &access->obj); - return ERR_PTR(rc); } EXPORT_SYMBOL_NS_GPL(iommufd_access_create, IOMMUFD); @@ -494,6 +463,30 @@ void iommufd_access_destroy(struct iommufd_access *access) } EXPORT_SYMBOL_NS_GPL(iommufd_access_destroy, IOMMUFD); +int iommufd_access_attach(struct iommufd_access *access, u32 ioas_id) +{ + struct iommufd_ioas *new_ioas; + int rc = 0; + + if (access->ioas) + return -EINVAL; + + new_ioas = iommufd_get_ioas(access->ictx, ioas_id); + if (IS_ERR(new_ioas)) + return PTR_ERR(new_ioas); + + rc = iopt_add_access(&new_ioas->iopt, access); + if (rc) { + iommufd_put_object(&new_ioas->obj); + return rc; + } + iommufd_ref_to_users(&new_ioas->obj); + + access->ioas = new_ioas; + return 0; +} +EXPORT_SYMBOL_NS_GPL(iommufd_access_attach, IOMMUFD); + /** * iommufd_access_notify_unmap - Notify users of an iopt to stop using it * @iopt: iopt to work on @@ -726,41 +719,3 @@ err_out: return rc; } EXPORT_SYMBOL_NS_GPL(iommufd_access_rw, IOMMUFD); - -#ifdef CONFIG_IOMMUFD_TEST -/* - * Creating a real iommufd_device is too hard, bypass creating a iommufd_device - * and go directly to attaching a domain. - */ -struct iommufd_hw_pagetable * -iommufd_device_selftest_attach(struct iommufd_ctx *ictx, - struct iommufd_ioas *ioas, - struct device *mock_dev) -{ - struct iommufd_hw_pagetable *hwpt; - int rc; - - hwpt = iommufd_hw_pagetable_alloc(ictx, ioas, mock_dev); - if (IS_ERR(hwpt)) - return hwpt; - - rc = iopt_table_add_domain(&hwpt->ioas->iopt, hwpt->domain); - if (rc) - goto out_hwpt; - - refcount_inc(&hwpt->obj.users); - iommufd_object_finalize(ictx, &hwpt->obj); - return hwpt; - -out_hwpt: - iommufd_object_abort_and_destroy(ictx, &hwpt->obj); - return ERR_PTR(rc); -} - -void iommufd_device_selftest_detach(struct iommufd_ctx *ictx, - struct iommufd_hw_pagetable *hwpt) -{ - iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain); - refcount_dec(&hwpt->obj.users); -} -#endif diff --git a/drivers/iommu/iommufd/hw_pagetable.c b/drivers/iommu/iommufd/hw_pagetable.c index 43d473989a06..6cdb6749d359 100644 --- a/drivers/iommu/iommufd/hw_pagetable.c +++ b/drivers/iommu/iommufd/hw_pagetable.c @@ -13,7 +13,17 @@ void iommufd_hw_pagetable_destroy(struct iommufd_object *obj) WARN_ON(!list_empty(&hwpt->devices)); - iommu_domain_free(hwpt->domain); + if (!list_empty(&hwpt->hwpt_item)) { + mutex_lock(&hwpt->ioas->mutex); + list_del(&hwpt->hwpt_item); + mutex_unlock(&hwpt->ioas->mutex); + + iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain); + } + + if (hwpt->domain) + iommu_domain_free(hwpt->domain); + refcount_dec(&hwpt->ioas->obj.users); mutex_destroy(&hwpt->devices_lock); } @@ -22,36 +32,74 @@ void iommufd_hw_pagetable_destroy(struct iommufd_object *obj) * iommufd_hw_pagetable_alloc() - Get an iommu_domain for a device * @ictx: iommufd context * @ioas: IOAS to associate the domain with - * @dev: Device to get an iommu_domain for + * @idev: Device to get an iommu_domain for + * @immediate_attach: True if idev should be attached to the hwpt * - * Allocate a new iommu_domain and return it as a hw_pagetable. + * Allocate a new iommu_domain and return it as a hw_pagetable. The HWPT + * will be linked to the given ioas and upon return the underlying iommu_domain + * is fully popoulated. */ struct iommufd_hw_pagetable * iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas, - struct device *dev) + struct iommufd_device *idev, bool immediate_attach) { struct iommufd_hw_pagetable *hwpt; int rc; + lockdep_assert_held(&ioas->mutex); + hwpt = iommufd_object_alloc(ictx, hwpt, IOMMUFD_OBJ_HW_PAGETABLE); if (IS_ERR(hwpt)) return hwpt; - hwpt->domain = iommu_domain_alloc(dev->bus); - if (!hwpt->domain) { - rc = -ENOMEM; - goto out_abort; - } - INIT_LIST_HEAD(&hwpt->devices); INIT_LIST_HEAD(&hwpt->hwpt_item); mutex_init(&hwpt->devices_lock); /* Pairs with iommufd_hw_pagetable_destroy() */ refcount_inc(&ioas->obj.users); hwpt->ioas = ioas; + + hwpt->domain = iommu_domain_alloc(idev->dev->bus); + if (!hwpt->domain) { + rc = -ENOMEM; + goto out_abort; + } + + mutex_lock(&hwpt->devices_lock); + + /* + * immediate_attach exists only to accommodate iommu drivers that cannot + * directly allocate a domain. These drivers do not finish creating the + * domain until attach is completed. Thus we must have this call + * sequence. Once those drivers are fixed this should be removed. + */ + if (immediate_attach) { + rc = iommufd_hw_pagetable_attach(hwpt, idev); + if (rc) + goto out_unlock; + } + + rc = iopt_table_add_domain(&hwpt->ioas->iopt, hwpt->domain); + if (rc) + goto out_detach; + list_add_tail(&hwpt->hwpt_item, &hwpt->ioas->hwpt_list); + + if (immediate_attach) { + /* See iommufd_device_do_attach() */ + refcount_inc(&hwpt->obj.users); + idev->hwpt = hwpt; + list_add(&idev->devices_item, &hwpt->devices); + } + + mutex_unlock(&hwpt->devices_lock); return hwpt; +out_detach: + if (immediate_attach) + iommufd_hw_pagetable_detach(hwpt, idev); +out_unlock: + mutex_unlock(&hwpt->devices_lock); out_abort: - iommufd_object_abort(ictx, &hwpt->obj); + iommufd_object_abort_and_destroy(ictx, &hwpt->obj); return ERR_PTR(rc); } diff --git a/drivers/iommu/iommufd/ioas.c b/drivers/iommu/iommufd/ioas.c index 31577e9d434f..d5624577f79f 100644 --- a/drivers/iommu/iommufd/ioas.c +++ b/drivers/iommu/iommufd/ioas.c @@ -71,7 +71,7 @@ int iommufd_ioas_iova_ranges(struct iommufd_ucmd *ucmd) if (cmd->__reserved) return -EOPNOTSUPP; - ioas = iommufd_get_ioas(ucmd, cmd->ioas_id); + ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); if (IS_ERR(ioas)) return PTR_ERR(ioas); @@ -151,7 +151,7 @@ int iommufd_ioas_allow_iovas(struct iommufd_ucmd *ucmd) if (cmd->__reserved) return -EOPNOTSUPP; - ioas = iommufd_get_ioas(ucmd, cmd->ioas_id); + ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); if (IS_ERR(ioas)) return PTR_ERR(ioas); iopt = &ioas->iopt; @@ -213,7 +213,7 @@ int iommufd_ioas_map(struct iommufd_ucmd *ucmd) if (cmd->iova >= ULONG_MAX || cmd->length >= ULONG_MAX) return -EOVERFLOW; - ioas = iommufd_get_ioas(ucmd, cmd->ioas_id); + ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); if (IS_ERR(ioas)) return PTR_ERR(ioas); @@ -253,7 +253,7 @@ int iommufd_ioas_copy(struct iommufd_ucmd *ucmd) cmd->dst_iova >= ULONG_MAX) return -EOVERFLOW; - src_ioas = iommufd_get_ioas(ucmd, cmd->src_ioas_id); + src_ioas = iommufd_get_ioas(ucmd->ictx, cmd->src_ioas_id); if (IS_ERR(src_ioas)) return PTR_ERR(src_ioas); rc = iopt_get_pages(&src_ioas->iopt, cmd->src_iova, cmd->length, @@ -262,7 +262,7 @@ int iommufd_ioas_copy(struct iommufd_ucmd *ucmd) if (rc) return rc; - dst_ioas = iommufd_get_ioas(ucmd, cmd->dst_ioas_id); + dst_ioas = iommufd_get_ioas(ucmd->ictx, cmd->dst_ioas_id); if (IS_ERR(dst_ioas)) { rc = PTR_ERR(dst_ioas); goto out_pages; @@ -292,7 +292,7 @@ int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd) unsigned long unmapped = 0; int rc; - ioas = iommufd_get_ioas(ucmd, cmd->ioas_id); + ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); if (IS_ERR(ioas)) return PTR_ERR(ioas); @@ -381,7 +381,7 @@ int iommufd_ioas_option(struct iommufd_ucmd *ucmd) if (cmd->__reserved) return -EOPNOTSUPP; - ioas = iommufd_get_ioas(ucmd, cmd->object_id); + ioas = iommufd_get_ioas(ucmd->ictx, cmd->object_id); if (IS_ERR(ioas)) return PTR_ERR(ioas); diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index 9d7f71510ca1..b38e67d1988b 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -12,6 +12,7 @@ struct iommu_domain; struct iommu_group; struct iommu_option; +struct iommufd_device; struct iommufd_ctx { struct file *file; @@ -211,10 +212,10 @@ struct iommufd_ioas { struct list_head hwpt_list; }; -static inline struct iommufd_ioas *iommufd_get_ioas(struct iommufd_ucmd *ucmd, +static inline struct iommufd_ioas *iommufd_get_ioas(struct iommufd_ctx *ictx, u32 id) { - return container_of(iommufd_get_object(ucmd->ictx, id, + return container_of(iommufd_get_object(ictx, id, IOMMUFD_OBJ_IOAS), struct iommufd_ioas, obj); } @@ -254,9 +255,30 @@ struct iommufd_hw_pagetable { struct iommufd_hw_pagetable * iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas, - struct device *dev); + struct iommufd_device *idev, bool immediate_attach); +int iommufd_hw_pagetable_attach(struct iommufd_hw_pagetable *hwpt, + struct iommufd_device *idev); +void iommufd_hw_pagetable_detach(struct iommufd_hw_pagetable *hwpt, + struct iommufd_device *idev); void iommufd_hw_pagetable_destroy(struct iommufd_object *obj); +/* + * A iommufd_device object represents the binding relationship between a + * consuming driver and the iommufd. These objects are created/destroyed by + * external drivers, not by userspace. + */ +struct iommufd_device { + struct iommufd_object obj; + struct iommufd_ctx *ictx; + struct iommufd_hw_pagetable *hwpt; + /* Head at iommufd_hw_pagetable::devices */ + struct list_head devices_item; + /* always the physical device */ + struct device *dev; + struct iommu_group *group; + bool enforce_cache_coherency; +}; + void iommufd_device_destroy(struct iommufd_object *obj); struct iommufd_access { @@ -275,12 +297,6 @@ void iopt_remove_access(struct io_pagetable *iopt, void iommufd_access_destroy_object(struct iommufd_object *obj); #ifdef CONFIG_IOMMUFD_TEST -struct iommufd_hw_pagetable * -iommufd_device_selftest_attach(struct iommufd_ctx *ictx, - struct iommufd_ioas *ioas, - struct device *mock_dev); -void iommufd_device_selftest_detach(struct iommufd_ctx *ictx, - struct iommufd_hw_pagetable *hwpt); int iommufd_test(struct iommufd_ucmd *ucmd); void iommufd_selftest_destroy(struct iommufd_object *obj); extern size_t iommufd_test_memory_limit; @@ -289,6 +305,7 @@ void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, bool iommufd_should_fail(void); void __init iommufd_test_init(void); void iommufd_test_exit(void); +bool iommufd_selftest_is_mock_dev(struct device *dev); #else static inline void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, unsigned int ioas_id, @@ -305,5 +322,9 @@ static inline void __init iommufd_test_init(void) static inline void iommufd_test_exit(void) { } +static inline bool iommufd_selftest_is_mock_dev(struct device *dev) +{ + return false; +} #endif #endif diff --git a/drivers/iommu/iommufd/iommufd_test.h b/drivers/iommu/iommufd/iommufd_test.h index 1d96a8f466fd..b3d69cca7729 100644 --- a/drivers/iommu/iommufd/iommufd_test.h +++ b/drivers/iommu/iommufd/iommufd_test.h @@ -49,7 +49,7 @@ struct iommu_test_cmd { __aligned_u64 length; } add_reserved; struct { - __u32 out_device_id; + __u32 out_stdev_id; __u32 out_hwpt_id; } mock_domain; struct { diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index f8d92c9bb65b..3c47846cc5ef 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -294,9 +294,9 @@ static void batch_clear_carry(struct pfn_batch *batch, unsigned int keep_pfns) batch->npfns[batch->end - 1] < keep_pfns); batch->total_pfns = keep_pfns; - batch->npfns[0] = keep_pfns; batch->pfns[0] = batch->pfns[batch->end - 1] + (batch->npfns[batch->end - 1] - keep_pfns); + batch->npfns[0] = keep_pfns; batch->end = 0; } @@ -1142,6 +1142,7 @@ struct iopt_pages *iopt_alloc_pages(void __user *uptr, unsigned long length, bool writable) { struct iopt_pages *pages; + unsigned long end; /* * The iommu API uses size_t as the length, and protect the DIV_ROUND_UP @@ -1150,6 +1151,9 @@ struct iopt_pages *iopt_alloc_pages(void __user *uptr, unsigned long length, if (length > SIZE_MAX - PAGE_SIZE || length == 0) return ERR_PTR(-EINVAL); + if (check_add_overflow((unsigned long)uptr, length, &end)) + return ERR_PTR(-EOVERFLOW); + pages = kzalloc(sizeof(*pages), GFP_KERNEL_ACCOUNT); if (!pages) return ERR_PTR(-ENOMEM); @@ -1203,13 +1207,21 @@ iopt_area_unpin_domain(struct pfn_batch *batch, struct iopt_area *area, unsigned long start = max(start_index, *unmapped_end_index); + if (IS_ENABLED(CONFIG_IOMMUFD_TEST) && + batch->total_pfns) + WARN_ON(*unmapped_end_index - + batch->total_pfns != + start_index); batch_from_domain(batch, domain, area, start, last_index); - batch_last_index = start + batch->total_pfns - 1; + batch_last_index = start_index + batch->total_pfns - 1; } else { batch_last_index = last_index; } + if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) + WARN_ON(batch_last_index > real_last_index); + /* * unmaps must always 'cut' at a place where the pfns are not * contiguous to pair with the maps that always install diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c index cfb5fe9a5e0e..74c2076105d4 100644 --- a/drivers/iommu/iommufd/selftest.c +++ b/drivers/iommu/iommufd/selftest.c @@ -75,7 +75,7 @@ void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, return; *flags &= ~(u32)MOCK_FLAGS_ACCESS_SYZ; - ioas = iommufd_get_ioas(ucmd, ioas_id); + ioas = iommufd_get_ioas(ucmd->ictx, ioas_id); if (IS_ERR(ioas)) return; *iova = iommufd_test_syz_conv_iova(&ioas->iopt, iova); @@ -91,23 +91,50 @@ enum selftest_obj_type { TYPE_IDEV, }; +struct mock_dev { + struct device dev; +}; + struct selftest_obj { struct iommufd_object obj; enum selftest_obj_type type; union { struct { - struct iommufd_hw_pagetable *hwpt; + struct iommufd_device *idev; struct iommufd_ctx *ictx; - struct device mock_dev; + struct mock_dev *mock_dev; } idev; }; }; +static void mock_domain_blocking_free(struct iommu_domain *domain) +{ +} + +static int mock_domain_nop_attach(struct iommu_domain *domain, + struct device *dev) +{ + return 0; +} + +static const struct iommu_domain_ops mock_blocking_ops = { + .free = mock_domain_blocking_free, + .attach_dev = mock_domain_nop_attach, +}; + +static struct iommu_domain mock_blocking_domain = { + .type = IOMMU_DOMAIN_BLOCKED, + .ops = &mock_blocking_ops, +}; + static struct iommu_domain *mock_domain_alloc(unsigned int iommu_domain_type) { struct mock_iommu_domain *mock; + if (iommu_domain_type == IOMMU_DOMAIN_BLOCKED) + return &mock_blocking_domain; + if (WARN_ON(iommu_domain_type != IOMMU_DOMAIN_UNMANAGED)) return NULL; @@ -236,19 +263,39 @@ static phys_addr_t mock_domain_iova_to_phys(struct iommu_domain *domain, return (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE; } +static bool mock_domain_capable(struct device *dev, enum iommu_cap cap) +{ + return cap == IOMMU_CAP_CACHE_COHERENCY; +} + +static void mock_domain_set_plaform_dma_ops(struct device *dev) +{ + /* + * mock doesn't setup default domains because we can't hook into the + * normal probe path + */ +} + static const struct iommu_ops mock_ops = { .owner = THIS_MODULE, .pgsize_bitmap = MOCK_IO_PAGE_SIZE, .domain_alloc = mock_domain_alloc, + .capable = mock_domain_capable, + .set_platform_dma_ops = mock_domain_set_plaform_dma_ops, .default_domain_ops = &(struct iommu_domain_ops){ .free = mock_domain_free, + .attach_dev = mock_domain_nop_attach, .map_pages = mock_domain_map_pages, .unmap_pages = mock_domain_unmap_pages, .iova_to_phys = mock_domain_iova_to_phys, }, }; +static struct iommu_device mock_iommu_device = { + .ops = &mock_ops, +}; + static inline struct iommufd_hw_pagetable * get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, struct mock_iommu_domain **mock) @@ -269,48 +316,142 @@ get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, return hwpt; } +static struct bus_type iommufd_mock_bus_type = { + .name = "iommufd_mock", + .iommu_ops = &mock_ops, +}; + +static void mock_dev_release(struct device *dev) +{ + struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); + + kfree(mdev); +} + +static struct mock_dev *mock_dev_create(void) +{ + struct iommu_group *iommu_group; + struct dev_iommu *dev_iommu; + struct mock_dev *mdev; + int rc; + + mdev = kzalloc(sizeof(*mdev), GFP_KERNEL); + if (!mdev) + return ERR_PTR(-ENOMEM); + + device_initialize(&mdev->dev); + mdev->dev.release = mock_dev_release; + mdev->dev.bus = &iommufd_mock_bus_type; + + iommu_group = iommu_group_alloc(); + if (IS_ERR(iommu_group)) { + rc = PTR_ERR(iommu_group); + goto err_put; + } + + rc = dev_set_name(&mdev->dev, "iommufd_mock%u", + iommu_group_id(iommu_group)); + if (rc) + goto err_group; + + /* + * The iommu core has no way to associate a single device with an iommu + * driver (heck currently it can't even support two iommu_drivers + * registering). Hack it together with an open coded dev_iommu_get(). + * Notice that the normal notifier triggered iommu release process also + * does not work here because this bus is not in iommu_buses. + */ + mdev->dev.iommu = kzalloc(sizeof(*dev_iommu), GFP_KERNEL); + if (!mdev->dev.iommu) { + rc = -ENOMEM; + goto err_group; + } + mutex_init(&mdev->dev.iommu->lock); + mdev->dev.iommu->iommu_dev = &mock_iommu_device; + + rc = device_add(&mdev->dev); + if (rc) + goto err_dev_iommu; + + rc = iommu_group_add_device(iommu_group, &mdev->dev); + if (rc) + goto err_del; + iommu_group_put(iommu_group); + return mdev; + +err_del: + device_del(&mdev->dev); +err_dev_iommu: + kfree(mdev->dev.iommu); + mdev->dev.iommu = NULL; +err_group: + iommu_group_put(iommu_group); +err_put: + put_device(&mdev->dev); + return ERR_PTR(rc); +} + +static void mock_dev_destroy(struct mock_dev *mdev) +{ + iommu_group_remove_device(&mdev->dev); + device_del(&mdev->dev); + kfree(mdev->dev.iommu); + mdev->dev.iommu = NULL; + put_device(&mdev->dev); +} + +bool iommufd_selftest_is_mock_dev(struct device *dev) +{ + return dev->release == mock_dev_release; +} + /* Create an hw_pagetable with the mock domain so we can test the domain ops */ static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd, struct iommu_test_cmd *cmd) { - static struct bus_type mock_bus = { .iommu_ops = &mock_ops }; - struct iommufd_hw_pagetable *hwpt; + struct iommufd_device *idev; struct selftest_obj *sobj; - struct iommufd_ioas *ioas; + u32 pt_id = cmd->id; + u32 idev_id; int rc; - ioas = iommufd_get_ioas(ucmd, cmd->id); - if (IS_ERR(ioas)) - return PTR_ERR(ioas); - sobj = iommufd_object_alloc(ucmd->ictx, sobj, IOMMUFD_OBJ_SELFTEST); - if (IS_ERR(sobj)) { - rc = PTR_ERR(sobj); - goto out_ioas; - } + if (IS_ERR(sobj)) + return PTR_ERR(sobj); + sobj->idev.ictx = ucmd->ictx; sobj->type = TYPE_IDEV; - sobj->idev.mock_dev.bus = &mock_bus; - hwpt = iommufd_device_selftest_attach(ucmd->ictx, ioas, - &sobj->idev.mock_dev); - if (IS_ERR(hwpt)) { - rc = PTR_ERR(hwpt); + sobj->idev.mock_dev = mock_dev_create(); + if (IS_ERR(sobj->idev.mock_dev)) { + rc = PTR_ERR(sobj->idev.mock_dev); goto out_sobj; } - sobj->idev.hwpt = hwpt; - /* Userspace must destroy both of these IDs to destroy the object */ - cmd->mock_domain.out_hwpt_id = hwpt->obj.id; - cmd->mock_domain.out_device_id = sobj->obj.id; + idev = iommufd_device_bind(ucmd->ictx, &sobj->idev.mock_dev->dev, + &idev_id); + if (IS_ERR(idev)) { + rc = PTR_ERR(idev); + goto out_mdev; + } + sobj->idev.idev = idev; + + rc = iommufd_device_attach(idev, &pt_id); + if (rc) + goto out_unbind; + + /* Userspace must destroy the device_id to destroy the object */ + cmd->mock_domain.out_hwpt_id = pt_id; + cmd->mock_domain.out_stdev_id = sobj->obj.id; iommufd_object_finalize(ucmd->ictx, &sobj->obj); - iommufd_put_object(&ioas->obj); return iommufd_ucmd_respond(ucmd, sizeof(*cmd)); +out_unbind: + iommufd_device_unbind(idev); +out_mdev: + mock_dev_destroy(sobj->idev.mock_dev); out_sobj: iommufd_object_abort(ucmd->ictx, &sobj->obj); -out_ioas: - iommufd_put_object(&ioas->obj); return rc; } @@ -322,7 +463,7 @@ static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd, struct iommufd_ioas *ioas; int rc; - ioas = iommufd_get_ioas(ucmd, mockpt_id); + ioas = iommufd_get_ioas(ucmd->ictx, mockpt_id); if (IS_ERR(ioas)) return PTR_ERR(ioas); down_write(&ioas->iopt.iova_rwsem); @@ -339,10 +480,12 @@ static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd, { struct iommufd_hw_pagetable *hwpt; struct mock_iommu_domain *mock; + uintptr_t end; int rc; if (iova % MOCK_IO_PAGE_SIZE || length % MOCK_IO_PAGE_SIZE || - (uintptr_t)uptr % MOCK_IO_PAGE_SIZE) + (uintptr_t)uptr % MOCK_IO_PAGE_SIZE || + check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) return -EINVAL; hwpt = get_md_pagetable(ucmd, mockpt_id, &mock); @@ -390,7 +533,10 @@ static int iommufd_test_md_check_refs(struct iommufd_ucmd *ucmd, void __user *uptr, size_t length, unsigned int refs) { - if (length % PAGE_SIZE || (uintptr_t)uptr % PAGE_SIZE) + uintptr_t end; + + if (length % PAGE_SIZE || (uintptr_t)uptr % PAGE_SIZE || + check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) return -EINVAL; for (; length; length -= PAGE_SIZE) { @@ -554,6 +700,7 @@ static int iommufd_test_create_access(struct iommufd_ucmd *ucmd, struct iommu_test_cmd *cmd = ucmd->cmd; struct selftest_access *staccess; struct iommufd_access *access; + u32 id; int fdno; int rc; @@ -571,15 +718,18 @@ static int iommufd_test_create_access(struct iommufd_ucmd *ucmd, } access = iommufd_access_create( - ucmd->ictx, ioas_id, + ucmd->ictx, (flags & MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) ? &selftest_access_ops_pin : &selftest_access_ops, - staccess); + staccess, &id); if (IS_ERR(access)) { rc = PTR_ERR(access); goto out_put_fdno; } + rc = iommufd_access_attach(access, ioas_id); + if (rc) + goto out_destroy; cmd->create_access.out_access_fd = fdno; rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); if (rc) @@ -780,8 +930,9 @@ void iommufd_selftest_destroy(struct iommufd_object *obj) switch (sobj->type) { case TYPE_IDEV: - iommufd_device_selftest_detach(sobj->idev.ictx, - sobj->idev.hwpt); + iommufd_device_detach(sobj->idev.idev); + iommufd_device_unbind(sobj->idev.idev); + mock_dev_destroy(sobj->idev.mock_dev); break; } } @@ -845,9 +996,11 @@ void __init iommufd_test_init(void) { dbgfs_root = fault_create_debugfs_attr("fail_iommufd", NULL, &fail_iommufd); + WARN_ON(bus_register(&iommufd_mock_bus_type)); } void iommufd_test_exit(void) { debugfs_remove_recursive(dbgfs_root); + bus_unregister(&iommufd_mock_bus_type); } diff --git a/drivers/iommu/iommufd/vfio_compat.c b/drivers/iommu/iommufd/vfio_compat.c index 514494a0025b..fe02517c73cc 100644 --- a/drivers/iommu/iommufd/vfio_compat.c +++ b/drivers/iommu/iommufd/vfio_compat.c @@ -137,7 +137,7 @@ int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd) return iommufd_ucmd_respond(ucmd, sizeof(*cmd)); case IOMMU_VFIO_IOAS_SET: - ioas = iommufd_get_ioas(ucmd, cmd->ioas_id); + ioas = iommufd_get_ioas(ucmd->ictx, cmd->ioas_id); if (IS_ERR(ioas)) return PTR_ERR(ioas); xa_lock(&ucmd->ictx->objects); diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c index 2d993d0cea7d..74c5cb93e900 100644 --- a/drivers/iommu/sun50i-iommu.c +++ b/drivers/iommu/sun50i-iommu.c @@ -1076,4 +1076,3 @@ builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe); MODULE_DESCRIPTION("Allwinner H6 IOMMU driver"); MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>"); MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>"); -MODULE_LICENSE("Dual BSD/GPL"); |