From 4f609dbff51b3c98d5fc841758a20be74ab3f132 Mon Sep 17 00:00:00 2001 From: Lu Baolu Date: Wed, 24 Apr 2024 15:16:41 +0800 Subject: iommu/vt-d: Use cache helpers in arch_invalidate_secondary_tlbs The arch_invalidate_secondary_tlbs callback is called in the SVA mm notification path. It invalidates all or a range of caches after the CPU page table is modified. Use the cache tag helps in this path. The mm_types defines vm_end as the first byte after the end address which is different from the iommu gather API, hence convert the end parameter from mm_types to iommu gather scheme before calling the cache_tag helper. Signed-off-by: Lu Baolu Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20240416080656.60968-10-baolu.lu@linux.intel.com Signed-off-by: Joerg Roedel --- drivers/iommu/intel/iommu.h | 1 + drivers/iommu/intel/svm.c | 81 ++++++--------------------------------------- 2 files changed, 11 insertions(+), 71 deletions(-) (limited to 'drivers/iommu/intel') diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 1d705a983dd7..fc0b4b048635 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1166,6 +1166,7 @@ struct intel_svm { struct mm_struct *mm; u32 pasid; struct list_head devs; + struct dmar_domain *domain; }; #else static inline void intel_svm_check(struct intel_iommu *iommu) {} diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index 2e627fbd5adb..5ba0a7baa455 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -168,88 +168,25 @@ void intel_svm_check(struct intel_iommu *iommu) iommu->flags |= VTD_FLAG_SVM_CAPABLE; } -static void __flush_svm_range_dev(struct intel_svm *svm, - struct intel_svm_dev *sdev, - unsigned long address, - unsigned long pages, int ih) -{ - struct device_domain_info *info = dev_iommu_priv_get(sdev->dev); - - if (WARN_ON(!pages)) - return; - - qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, address, pages, ih); - if (info->ats_enabled) { - qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid, - svm->pasid, sdev->qdep, address, - order_base_2(pages)); - quirk_extra_dev_tlb_flush(info, address, order_base_2(pages), - svm->pasid, sdev->qdep); - } -} - -static void intel_flush_svm_range_dev(struct intel_svm *svm, - struct intel_svm_dev *sdev, - unsigned long address, - unsigned long pages, int ih) -{ - unsigned long shift = ilog2(__roundup_pow_of_two(pages)); - unsigned long align = (1ULL << (VTD_PAGE_SHIFT + shift)); - unsigned long start = ALIGN_DOWN(address, align); - unsigned long end = ALIGN(address + (pages << VTD_PAGE_SHIFT), align); - - while (start < end) { - __flush_svm_range_dev(svm, sdev, start, align >> VTD_PAGE_SHIFT, ih); - start += align; - } -} - -static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address, - unsigned long pages, int ih) -{ - struct intel_svm_dev *sdev; - - rcu_read_lock(); - list_for_each_entry_rcu(sdev, &svm->devs, list) - intel_flush_svm_range_dev(svm, sdev, address, pages, ih); - rcu_read_unlock(); -} - -static void intel_flush_svm_all(struct intel_svm *svm) -{ - struct device_domain_info *info; - struct intel_svm_dev *sdev; - - rcu_read_lock(); - list_for_each_entry_rcu(sdev, &svm->devs, list) { - info = dev_iommu_priv_get(sdev->dev); - - qi_flush_piotlb(sdev->iommu, sdev->did, svm->pasid, 0, -1UL, 0); - if (info->ats_enabled) { - qi_flush_dev_iotlb_pasid(sdev->iommu, sdev->sid, info->pfsid, - svm->pasid, sdev->qdep, - 0, 64 - VTD_PAGE_SHIFT); - quirk_extra_dev_tlb_flush(info, 0, 64 - VTD_PAGE_SHIFT, - svm->pasid, sdev->qdep); - } - } - rcu_read_unlock(); -} - /* Pages have been freed at this point */ static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, struct mm_struct *mm, unsigned long start, unsigned long end) { struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); + struct dmar_domain *domain = svm->domain; if (start == 0 && end == -1UL) { - intel_flush_svm_all(svm); + cache_tag_flush_all(domain); return; } - intel_flush_svm_range(svm, start, - (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0); + /* + * The mm_types defines vm_end as the first byte after the end address, + * different from IOMMU subsystem using the last address of an address + * range. + */ + cache_tag_flush_range(domain, start, end - 1, 0); } static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) @@ -336,6 +273,7 @@ static int intel_svm_set_dev_pasid(struct iommu_domain *domain, INIT_LIST_HEAD_RCU(&svm->devs); svm->notifier.ops = &intel_mmuops; + svm->domain = to_dmar_domain(domain); ret = mmu_notifier_register(&svm->notifier, mm); if (ret) { kfree(svm); @@ -747,6 +685,7 @@ struct iommu_domain *intel_svm_domain_alloc(void) if (!domain) return NULL; domain->domain.ops = &intel_svm_domain_ops; + domain->use_first_level = true; INIT_LIST_HEAD(&domain->cache_tags); spin_lock_init(&domain->cache_lock); -- cgit v1.2.3