From 0d571dcbe7c6d36dcfcb8e04a49cc01fe462d171 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Tue, 21 Mar 2023 09:23:47 +0000 Subject: iommu/amd: Allocate page table using numa locality info Introduce 'struct protection_domain->nid' variable. It will contain IOMMU NUMA node ID. And allocate page table pages using IOMMU numa locality info. This optimizes page table walk by IOMMU. Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20230321092348.6127-2-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/iommu/amd/amd_iommu_types.h') diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 3d684190b4d5..e5cfdeaaad48 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -549,6 +549,7 @@ struct protection_domain { spinlock_t lock; /* mostly used to lock the page table*/ u16 id; /* the domain id written to the device table */ int glx; /* Number of levels for GCR3 table */ + int nid; /* Node ID */ u64 *gcr3_tbl; /* Guest CR3 table */ unsigned long flags; /* flags to find out type of domain */ unsigned dev_cnt; /* devices assigned to this domain */ -- cgit v1.2.3 From f594496403fa383259aa7dfad92f383a2ee07e1b Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Fri, 10 Mar 2023 09:00:00 +0000 Subject: iommu/amd: Add 5 level guest page table support Newer AMD IOMMU supports 5 level guest page table (v2 page table). If both processor and IOMMU supports 5 level page table then enable it. Otherwise fall back to 4 level page table. Co-developed-by: Wei Huang Signed-off-by: Wei Huang Reviewed-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20230310090000.1117786-1-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu.h | 1 + drivers/iommu/amd/amd_iommu_types.h | 7 +++++++ drivers/iommu/amd/init.c | 23 +++++++++++++++++++++-- drivers/iommu/amd/io_pgtable_v2.c | 9 ++++++--- drivers/iommu/amd/iommu.c | 5 +++++ 5 files changed, 40 insertions(+), 5 deletions(-) (limited to 'drivers/iommu/amd/amd_iommu_types.h') diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 20a142b54498..e98f20a9bdd8 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -34,6 +34,7 @@ extern int amd_iommu_reenable(int); extern int amd_iommu_enable_faulting(void); extern int amd_iommu_guest_ir; extern enum io_pgtable_fmt amd_iommu_pgtable; +extern int amd_iommu_gpt_level; /* IOMMUv2 specific functions */ struct iommu_domain; diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index e5cfdeaaad48..3cda781b6c7f 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -93,6 +93,8 @@ #define FEATURE_GA (1ULL<<7) #define FEATURE_HE (1ULL<<8) #define FEATURE_PC (1ULL<<9) +#define FEATURE_GATS_SHIFT (12) +#define FEATURE_GATS_MASK (3ULL) #define FEATURE_GAM_VAPIC (1ULL<<21) #define FEATURE_GIOSUP (1ULL<<48) #define FEATURE_EPHSUP (1ULL<<50) @@ -305,6 +307,9 @@ #define PAGE_MODE_6_LEVEL 0x06 #define PAGE_MODE_7_LEVEL 0x07 +#define GUEST_PGTABLE_4_LEVEL 0x00 +#define GUEST_PGTABLE_5_LEVEL 0x01 + #define PM_LEVEL_SHIFT(x) (12 + ((x) * 9)) #define PM_LEVEL_SIZE(x) (((x) < 6) ? \ ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \ @@ -398,6 +403,8 @@ #define DTE_GCR3_SHIFT_B 16 #define DTE_GCR3_SHIFT_C 43 +#define DTE_GPT_LEVEL_SHIFT 54 + #define GCR3_VALID 0x01ULL #define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 141eca29c951..c46ee218d387 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -153,6 +153,8 @@ bool amd_iommu_dump; bool amd_iommu_irq_remap __read_mostly; enum io_pgtable_fmt amd_iommu_pgtable = AMD_IOMMU_V1; +/* Guest page table level */ +int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL; int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; @@ -306,6 +308,11 @@ static bool check_feature_on_all_iommus(u64 mask) return !!(amd_iommu_efr & mask); } +static inline int check_feature_gpt_level(void) +{ + return ((amd_iommu_efr >> FEATURE_GATS_SHIFT) & FEATURE_GATS_MASK); +} + /* * For IVHD type 0x11/0x40, EFR is also available via IVHD. * Default to IVHD EFR since it is available sooner @@ -2155,8 +2162,10 @@ static void print_iommu_info(void) if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) pr_info("X2APIC enabled\n"); } - if (amd_iommu_pgtable == AMD_IOMMU_V2) - pr_info("V2 page table enabled\n"); + if (amd_iommu_pgtable == AMD_IOMMU_V2) { + pr_info("V2 page table enabled (Paging mode : %d level)\n", + amd_iommu_gpt_level); + } } static int __init amd_iommu_init_pci(void) @@ -3026,6 +3035,11 @@ static int __init early_amd_iommu_init(void) if (ret) goto out; + /* 5 level guest page table */ + if (cpu_feature_enabled(X86_FEATURE_LA57) && + check_feature_gpt_level() == GUEST_PGTABLE_5_LEVEL) + amd_iommu_gpt_level = PAGE_MODE_5_LEVEL; + /* Disable any previously enabled IOMMUs */ if (!is_kdump_kernel() || amd_iommu_disabled) disable_iommus(); @@ -3557,6 +3571,11 @@ __setup("ivrs_acpihid", parse_ivrs_acpihid); bool amd_iommu_v2_supported(void) { + /* CPU page table size should match IOMMU guest page table size */ + if (cpu_feature_enabled(X86_FEATURE_LA57) && + amd_iommu_gpt_level != PAGE_MODE_5_LEVEL) + return false; + /* * Since DTE[Mode]=0 is prohibited on SNP-enabled system * (i.e. EFR[SNPSup]=1), IOMMUv2 page table cannot be used without diff --git a/drivers/iommu/amd/io_pgtable_v2.c b/drivers/iommu/amd/io_pgtable_v2.c index 9f8b423f7630..27c3015947e6 100644 --- a/drivers/iommu/amd/io_pgtable_v2.c +++ b/drivers/iommu/amd/io_pgtable_v2.c @@ -37,8 +37,7 @@ static inline int get_pgtable_level(void) { - /* 5 level page table is not supported */ - return PAGE_MODE_4_LEVEL; + return amd_iommu_gpt_level; } static inline bool is_large_pte(u64 pte) @@ -379,6 +378,7 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg); struct protection_domain *pdom = (struct protection_domain *)cookie; int ret; + int ias = IOMMU_IN_ADDR_BIT_SIZE; pgtable->pgd = alloc_pgtable_page(pdom->nid, GFP_ATOMIC); if (!pgtable->pgd) @@ -388,12 +388,15 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo if (ret) goto err_free_pgd; + if (get_pgtable_level() == PAGE_MODE_5_LEVEL) + ias = 57; + pgtable->iop.ops.map_pages = iommu_v2_map_pages; pgtable->iop.ops.unmap_pages = iommu_v2_unmap_pages; pgtable->iop.ops.iova_to_phys = iommu_v2_iova_to_phys; cfg->pgsize_bitmap = AMD_IOMMU_PGSIZES_V2, - cfg->ias = IOMMU_IN_ADDR_BIT_SIZE, + cfg->ias = ias, cfg->oas = IOMMU_OUT_ADDR_BIT_SIZE, cfg->tlb = &v2_flush_ops; diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 5452d0dd6569..e290f2034252 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1611,6 +1611,11 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid, tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C; flags |= tmp; + if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) { + dev_table[devid].data[2] |= + ((u64)GUEST_PGTABLE_5_LEVEL << DTE_GPT_LEVEL_SHIFT); + } + if (domain->flags & PD_GIOV_MASK) pte_root |= DTE_FLAG_GIOV; } -- cgit v1.2.3