diff options
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_buffer.c | 10 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_drv.c | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_drv.h | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_dump.c | 12 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_gem.c | 24 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_gem.h | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 31 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_gpu.h | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_iommu.c | 151 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_iommu.h | 20 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c | 264 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_mmu.c | 259 | ||||
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_mmu.h | 91 |
15 files changed, 461 insertions, 430 deletions
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c index a3cdb20bfc5f..4324b098689f 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c @@ -207,7 +207,7 @@ u16 etnaviv_buffer_config_mmuv2(struct etnaviv_gpu *gpu, u32 mtlb_addr, u32 safe return buffer->user_size / 8; } -u16 etnaviv_buffer_config_pta(struct etnaviv_gpu *gpu) +u16 etnaviv_buffer_config_pta(struct etnaviv_gpu *gpu, unsigned short id) { struct etnaviv_cmdbuf *buffer = &gpu->buffer; @@ -216,7 +216,7 @@ u16 etnaviv_buffer_config_pta(struct etnaviv_gpu *gpu) buffer->user_size = 0; CMD_LOAD_STATE(buffer, VIVS_MMUv2_PTA_CONFIG, - VIVS_MMUv2_PTA_CONFIG_INDEX(0)); + VIVS_MMUv2_PTA_CONFIG_INDEX(id)); CMD_END(buffer); @@ -315,7 +315,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state, u32 return_target, return_dwords; u32 link_target, link_dwords; bool switch_context = gpu->exec_state != exec_state; - unsigned int new_flush_seq = READ_ONCE(gpu->mmu->flush_seq); + unsigned int new_flush_seq = READ_ONCE(gpu->mmu_context->flush_seq); bool need_flush = gpu->flush_seq != new_flush_seq; lockdep_assert_held(&gpu->lock); @@ -339,7 +339,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state, /* flush command */ if (need_flush) { - if (gpu->mmu->version == ETNAVIV_IOMMU_V1) + if (gpu->mmu_context->global->version == ETNAVIV_IOMMU_V1) extra_dwords += 1; else extra_dwords += 3; @@ -353,7 +353,7 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state, if (need_flush) { /* Add the MMU flush */ - if (gpu->mmu->version == ETNAVIV_IOMMU_V1) { + if (gpu->mmu_context->global->version == ETNAVIV_IOMMU_V1) { CMD_LOAD_STATE(buffer, VIVS_GL_FLUSH_MMU, VIVS_GL_FLUSH_MMU_FLUSH_FEMMU | VIVS_GL_FLUSH_MMU_FLUSH_UNK1 | diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c index b0babc0f7230..f39430ba3593 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.c @@ -60,18 +60,18 @@ free_suballoc: } int etnaviv_cmdbuf_suballoc_map(struct etnaviv_cmdbuf_suballoc *suballoc, - struct etnaviv_iommu *mmu, + struct etnaviv_iommu_context *context, struct etnaviv_vram_mapping *mapping, u32 memory_base) { - return etnaviv_iommu_get_suballoc_va(mmu, mapping, memory_base, + return etnaviv_iommu_get_suballoc_va(context, mapping, memory_base, suballoc->paddr, SUBALLOC_SIZE); } -void etnaviv_cmdbuf_suballoc_unmap(struct etnaviv_iommu *mmu, +void etnaviv_cmdbuf_suballoc_unmap(struct etnaviv_iommu_context *context, struct etnaviv_vram_mapping *mapping) { - etnaviv_iommu_put_suballoc_va(mmu, mapping); + etnaviv_iommu_put_suballoc_va(context, mapping); } void etnaviv_cmdbuf_suballoc_destroy(struct etnaviv_cmdbuf_suballoc *suballoc) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h index a28668e46e26..ad6fd8eb0378 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_cmdbuf.h @@ -9,7 +9,7 @@ #include <linux/types.h> struct device; -struct etnaviv_iommu; +struct etnaviv_iommu_context; struct etnaviv_vram_mapping; struct etnaviv_cmdbuf_suballoc; struct etnaviv_perfmon_request; @@ -28,10 +28,10 @@ struct etnaviv_cmdbuf_suballoc * etnaviv_cmdbuf_suballoc_new(struct device *dev); void etnaviv_cmdbuf_suballoc_destroy(struct etnaviv_cmdbuf_suballoc *suballoc); int etnaviv_cmdbuf_suballoc_map(struct etnaviv_cmdbuf_suballoc *suballoc, - struct etnaviv_iommu *mmu, + struct etnaviv_iommu_context *context, struct etnaviv_vram_mapping *mapping, u32 memory_base); -void etnaviv_cmdbuf_suballoc_unmap(struct etnaviv_iommu *mmu, +void etnaviv_cmdbuf_suballoc_unmap(struct etnaviv_iommu_context *context, struct etnaviv_vram_mapping *mapping); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c index 5fa3aa7bdbc5..eb0c23fe979a 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c @@ -119,9 +119,9 @@ static int etnaviv_mmu_show(struct etnaviv_gpu *gpu, struct seq_file *m) seq_printf(m, "Active Objects (%s):\n", dev_name(gpu->dev)); - mutex_lock(&gpu->mmu->lock); - drm_mm_print(&gpu->mmu->mm, &p); - mutex_unlock(&gpu->mmu->lock); + mutex_lock(&gpu->mmu_context->lock); + drm_mm_print(&gpu->mmu_context->mm, &p); + mutex_unlock(&gpu->mmu_context->lock); return 0; } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h index e052d7db66ae..5f8db08f1c17 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h @@ -22,6 +22,7 @@ struct etnaviv_gpu; struct etnaviv_mmu; struct etnaviv_gem_object; struct etnaviv_gem_submit; +struct etnaviv_iommu_global; struct etnaviv_file_private { /* @@ -37,6 +38,7 @@ struct etnaviv_drm_private { struct etnaviv_gpu *gpu[ETNA_MAX_PIPES]; struct etnaviv_cmdbuf_suballoc *cmdbuf_suballoc; + struct etnaviv_iommu_global *mmu_global; /* list of GEM objects: */ struct mutex gem_lock; @@ -69,7 +71,7 @@ int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file, uintptr_t ptr, u32 size, u32 flags, u32 *handle); u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu); u16 etnaviv_buffer_config_mmuv2(struct etnaviv_gpu *gpu, u32 mtlb_addr, u32 safe_addr); -u16 etnaviv_buffer_config_pta(struct etnaviv_gpu *gpu); +u16 etnaviv_buffer_config_pta(struct etnaviv_gpu *gpu, unsigned short id); void etnaviv_buffer_end(struct etnaviv_gpu *gpu); void etnaviv_sync_point_queue(struct etnaviv_gpu *gpu, unsigned int event); void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state, diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.c b/drivers/gpu/drm/etnaviv/etnaviv_dump.c index 2ce60baa4ad9..7e6791517693 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_dump.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.c @@ -93,7 +93,7 @@ static void etnaviv_core_dump_registers(struct core_dump_iterator *iter, } static void etnaviv_core_dump_mmu(struct core_dump_iterator *iter, - struct etnaviv_iommu *mmu, size_t mmu_size) + struct etnaviv_iommu_context *mmu, size_t mmu_size) { etnaviv_iommu_dump(mmu, iter->data); @@ -125,9 +125,9 @@ void etnaviv_core_dump(struct etnaviv_gem_submit *submit) return; etnaviv_dump_core = false; - mutex_lock(&gpu->mmu->lock); + mutex_lock(&gpu->mmu_context->lock); - mmu_size = etnaviv_iommu_dump_size(gpu->mmu); + mmu_size = etnaviv_iommu_dump_size(gpu->mmu_context); /* We always dump registers, mmu, ring, hanging cmdbuf and end marker */ n_obj = 5; @@ -157,7 +157,7 @@ void etnaviv_core_dump(struct etnaviv_gem_submit *submit) iter.start = __vmalloc(file_size, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY, PAGE_KERNEL); if (!iter.start) { - mutex_unlock(&gpu->mmu->lock); + mutex_unlock(&gpu->mmu_context->lock); dev_warn(gpu->dev, "failed to allocate devcoredump file\n"); return; } @@ -169,7 +169,7 @@ void etnaviv_core_dump(struct etnaviv_gem_submit *submit) memset(iter.hdr, 0, iter.data - iter.start); etnaviv_core_dump_registers(&iter, gpu); - etnaviv_core_dump_mmu(&iter, gpu->mmu, mmu_size); + etnaviv_core_dump_mmu(&iter, gpu->mmu_context, mmu_size); etnaviv_core_dump_mem(&iter, ETDUMP_BUF_RING, gpu->buffer.vaddr, gpu->buffer.size, etnaviv_cmdbuf_get_va(&gpu->buffer, @@ -221,7 +221,7 @@ void etnaviv_core_dump(struct etnaviv_gem_submit *submit) obj->base.size); } - mutex_unlock(&gpu->mmu->lock); + mutex_unlock(&gpu->mmu_context->lock); etnaviv_core_dump_header(&iter, ETDUMP_BUF_END, iter.data); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index e199a6833ff0..0ccc3c4dffc4 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -223,12 +223,12 @@ int etnaviv_gem_mmap_offset(struct drm_gem_object *obj, u64 *offset) static struct etnaviv_vram_mapping * etnaviv_gem_get_vram_mapping(struct etnaviv_gem_object *obj, - struct etnaviv_iommu *mmu) + struct etnaviv_iommu_context *context) { struct etnaviv_vram_mapping *mapping; list_for_each_entry(mapping, &obj->vram_list, obj_node) { - if (mapping->mmu == mmu) + if (mapping->context == context) return mapping; } @@ -256,7 +256,7 @@ struct etnaviv_vram_mapping *etnaviv_gem_mapping_get( int ret = 0; mutex_lock(&etnaviv_obj->lock); - mapping = etnaviv_gem_get_vram_mapping(etnaviv_obj, gpu->mmu); + mapping = etnaviv_gem_get_vram_mapping(etnaviv_obj, gpu->mmu_context); if (mapping) { /* * Holding the object lock prevents the use count changing @@ -265,12 +265,12 @@ struct etnaviv_vram_mapping *etnaviv_gem_mapping_get( * the MMU owns this mapping to close this race. */ if (mapping->use == 0) { - mutex_lock(&gpu->mmu->lock); - if (mapping->mmu == gpu->mmu) + mutex_lock(&gpu->mmu_context->lock); + if (mapping->context == gpu->mmu_context) mapping->use += 1; else mapping = NULL; - mutex_unlock(&gpu->mmu->lock); + mutex_unlock(&gpu->mmu_context->lock); if (mapping) goto out; } else { @@ -303,11 +303,11 @@ struct etnaviv_vram_mapping *etnaviv_gem_mapping_get( list_del(&mapping->obj_node); } - mapping->mmu = gpu->mmu; + mapping->context = gpu->mmu_context; mapping->use = 1; - ret = etnaviv_iommu_map_gem(gpu->mmu, etnaviv_obj, gpu->memory_base, - mapping); + ret = etnaviv_iommu_map_gem(gpu->mmu_context, etnaviv_obj, + gpu->memory_base, mapping); if (ret < 0) kfree(mapping); else @@ -525,12 +525,12 @@ void etnaviv_gem_free_object(struct drm_gem_object *obj) list_for_each_entry_safe(mapping, tmp, &etnaviv_obj->vram_list, obj_node) { - struct etnaviv_iommu *mmu = mapping->mmu; + struct etnaviv_iommu_context *context = mapping->context; WARN_ON(mapping->use); - if (mmu) - etnaviv_iommu_unmap_gem(mmu, mapping); + if (context) + etnaviv_iommu_unmap_gem(context, mapping); list_del(&mapping->obj_node); kfree(mapping); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.h b/drivers/gpu/drm/etnaviv/etnaviv_gem.h index d7d8a835f379..5a004d5e4eaa 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.h @@ -25,7 +25,7 @@ struct etnaviv_vram_mapping { struct list_head scan_node; struct list_head mmu_node; struct etnaviv_gem_object *object; - struct etnaviv_iommu *mmu; + struct etnaviv_iommu_context *context; struct drm_mm_node vram_node; unsigned int use; u32 iova; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index 179bc6c544ca..885ca8f92338 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -681,7 +681,7 @@ static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu) etnaviv_gpu_setup_pulse_eater(gpu); /* setup the MMU */ - etnaviv_iommu_restore(gpu); + etnaviv_iommu_restore(gpu, gpu->mmu_context); /* Start command processor */ prefetch = etnaviv_buffer_init(gpu); @@ -754,14 +754,19 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu) goto fail; } - gpu->mmu = etnaviv_iommu_new(gpu); - if (IS_ERR(gpu->mmu)) { - dev_err(gpu->dev, "Failed to instantiate GPU IOMMU\n"); - ret = PTR_ERR(gpu->mmu); + ret = etnaviv_iommu_global_init(gpu); + if (ret) goto fail; + + gpu->mmu_context = etnaviv_iommu_context_init(priv->mmu_global); + if (IS_ERR(gpu->mmu_context)) { + dev_err(gpu->dev, "Failed to instantiate GPU IOMMU\n"); + ret = PTR_ERR(gpu->mmu_context); + goto iommu_global_fini; } - ret = etnaviv_cmdbuf_suballoc_map(priv->cmdbuf_suballoc, gpu->mmu, + ret = etnaviv_cmdbuf_suballoc_map(priv->cmdbuf_suballoc, + gpu->mmu_context, &gpu->cmdbuf_mapping, gpu->memory_base); if (ret) { @@ -777,7 +782,7 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu) goto unmap_suballoc; } - if (gpu->mmu->version == ETNAVIV_IOMMU_V1 && + if (!(gpu->identity.minor_features1 & chipMinorFeatures1_MMU_VERSION) && etnaviv_cmdbuf_get_va(&gpu->buffer, &gpu->cmdbuf_mapping) > 0x80000000) { ret = -EINVAL; dev_err(gpu->dev, @@ -808,9 +813,11 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu) free_buffer: etnaviv_cmdbuf_free(&gpu->buffer); unmap_suballoc: - etnaviv_cmdbuf_suballoc_unmap(gpu->mmu, &gpu->cmdbuf_mapping); + etnaviv_cmdbuf_suballoc_unmap(gpu->mmu_context, &gpu->cmdbuf_mapping); destroy_iommu: - etnaviv_iommu_destroy(gpu->mmu); + etnaviv_iommu_context_put(gpu->mmu_context); +iommu_global_fini: + etnaviv_iommu_global_fini(gpu); fail: pm_runtime_mark_last_busy(gpu->dev); pm_runtime_put_autosuspend(gpu->dev); @@ -1683,8 +1690,10 @@ static void etnaviv_gpu_unbind(struct device *dev, struct device *master, if (gpu->initialized) { etnaviv_cmdbuf_free(&gpu->buffer); - etnaviv_cmdbuf_suballoc_unmap(gpu->mmu, &gpu->cmdbuf_mapping); - etnaviv_iommu_destroy(gpu->mmu); + etnaviv_cmdbuf_suballoc_unmap(gpu->mmu_context, + &gpu->cmdbuf_mapping); + etnaviv_iommu_context_put(gpu->mmu_context); + etnaviv_iommu_global_fini(gpu); gpu->initialized = false; } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h index 96380942cd8c..c0bd6018d53b 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h @@ -8,6 +8,7 @@ #include "etnaviv_cmdbuf.h" #include "etnaviv_gem.h" +#include "etnaviv_mmu.h" #include "etnaviv_drv.h" struct etnaviv_gem_submit; @@ -136,7 +137,7 @@ struct etnaviv_gpu { void __iomem *mmio; int irq; - struct etnaviv_iommu *mmu; + struct etnaviv_iommu_context *mmu_context; unsigned int flush_seq; /* Power Control: */ diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c index 18c627c5cae1..a2f1ff151822 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_iommu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c @@ -11,7 +11,6 @@ #include "etnaviv_gpu.h" #include "etnaviv_mmu.h" -#include "etnaviv_iommu.h" #include "state_hi.xml.h" #define PT_SIZE SZ_2M @@ -19,113 +18,78 @@ #define GPU_MEM_START 0x80000000 -struct etnaviv_iommuv1_domain { - struct etnaviv_iommu_domain base; +struct etnaviv_iommuv1_context { + struct etnaviv_iommu_context base; u32 *pgtable_cpu; dma_addr_t pgtable_dma; }; -static struct etnaviv_iommuv1_domain * -to_etnaviv_domain(struct etnaviv_iommu_domain *domain) +static struct etnaviv_iommuv1_context * +to_v1_context(struct etnaviv_iommu_context *context) { - return container_of(domain, struct etnaviv_iommuv1_domain, base); + return container_of(context, struct etnaviv_iommuv1_context, base); } -static int __etnaviv_iommu_init(struct etnaviv_iommuv1_domain *etnaviv_domain) +static void etnaviv_iommuv1_free(struct etnaviv_iommu_context *context) { - u32 *p; - int i; - - etnaviv_domain->base.bad_page_cpu = - dma_alloc_wc(etnaviv_domain->base.dev, SZ_4K, - &etnaviv_domain->base.bad_page_dma, - GFP_KERNEL); - if (!etnaviv_domain->base.bad_page_cpu) - return -ENOMEM; - - p = etnaviv_domain->base.bad_page_cpu; - for (i = 0; i < SZ_4K / 4; i++) - *p++ = 0xdead55aa; - - etnaviv_domain->pgtable_cpu = dma_alloc_wc(etnaviv_domain->base.dev, - PT_SIZE, - &etnaviv_domain->pgtable_dma, - GFP_KERNEL); - if (!etnaviv_domain->pgtable_cpu) { - dma_free_wc(etnaviv_domain->base.dev, SZ_4K, - etnaviv_domain->base.bad_page_cpu, - etnaviv_domain->base.bad_page_dma); - return -ENOMEM; - } - - memset32(etnaviv_domain->pgtable_cpu, etnaviv_domain->base.bad_page_dma, - PT_ENTRIES); - - return 0; -} + struct etnaviv_iommuv1_context *v1_context = to_v1_context(context); -static void etnaviv_iommuv1_domain_free(struct etnaviv_iommu_domain *domain) -{ - struct etnaviv_iommuv1_domain *etnaviv_domain = - to_etnaviv_domain(domain); + drm_mm_takedown(&context->mm); - dma_free_wc(etnaviv_domain->base.dev, PT_SIZE, - etnaviv_domain->pgtable_cpu, etnaviv_domain->pgtable_dma); + dma_free_wc(context->global->dev, PT_SIZE, v1_context->pgtable_cpu, + v1_context->pgtable_dma); - dma_free_wc(etnaviv_domain->base.dev, SZ_4K, - etnaviv_domain->base.bad_page_cpu, - etnaviv_domain->base.bad_page_dma); + context->global->v1.shared_context = NULL; - kfree(etnaviv_domain); + kfree(v1_context); } -static int etnaviv_iommuv1_map(struct etnaviv_iommu_domain *domain, +static int etnaviv_iommuv1_map(struct etnaviv_iommu_context *context, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { - struct etnaviv_iommuv1_domain *etnaviv_domain = to_etnaviv_domain(domain); + struct etnaviv_iommuv1_context *v1_context = to_v1_context(context); unsigned int index = (iova - GPU_MEM_START) / SZ_4K; if (size != SZ_4K) return -EINVAL; - etnaviv_domain->pgtable_cpu[index] = paddr; + v1_context->pgtable_cpu[index] = paddr; return 0; } -static size_t etnaviv_iommuv1_unmap(struct etnaviv_iommu_domain *domain, +static size_t etnaviv_iommuv1_unmap(struct etnaviv_iommu_context *context, unsigned long iova, size_t size) { - struct etnaviv_iommuv1_domain *etnaviv_domain = - to_etnaviv_domain(domain); + struct etnaviv_iommuv1_context *v1_context = to_v1_context(context); unsigned int index = (iova - GPU_MEM_START) / SZ_4K; if (size != SZ_4K) return -EINVAL; - etnaviv_domain->pgtable_cpu[index] = etnaviv_domain->base.bad_page_dma; + v1_context->pgtable_cpu[index] = context->global->bad_page_dma; return SZ_4K; } -static size_t etnaviv_iommuv1_dump_size(struct etnaviv_iommu_domain *domain) +static size_t etnaviv_iommuv1_dump_size(struct etnaviv_iommu_context *context) { return PT_SIZE; } -static void etnaviv_iommuv1_dump(struct etnaviv_iommu_domain *domain, void *buf) +static void etnaviv_iommuv1_dump(struct etnaviv_iommu_context *context, + void *buf) { - struct etnaviv_iommuv1_domain *etnaviv_domain = - to_etnaviv_domain(domain); + struct etnaviv_iommuv1_context *v1_context = to_v1_context(context); - memcpy(buf, etnaviv_domain->pgtable_cpu, PT_SIZE); + memcpy(buf, v1_context->pgtable_cpu, PT_SIZE); } -void etnaviv_iommuv1_restore(struct etnaviv_gpu *gpu) +static void etnaviv_iommuv1_restore(struct etnaviv_gpu *gpu, + struct etnaviv_iommu_context *context) { - struct etnaviv_iommuv1_domain *etnaviv_domain = - to_etnaviv_domain(gpu->mmu->domain); + struct etnaviv_iommuv1_context *v1_context = to_v1_context(context); u32 pgtable; /* set base addresses */ @@ -136,7 +100,7 @@ void etnaviv_iommuv1_restore(struct etnaviv_gpu *gpu) gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_PE, gpu->memory_base); /* set page table address in MC */ - pgtable = (u32)etnaviv_domain->pgtable_dma; + pgtable = (u32)v1_context->pgtable_dma; gpu_write(gpu, VIVS_MC_MMU_FE_PAGE_TABLE, pgtable); gpu_write(gpu, VIVS_MC_MMU_TX_PAGE_TABLE, pgtable); @@ -145,39 +109,62 @@ void etnaviv_iommuv1_restore(struct etnaviv_gpu *gpu) gpu_write(gpu, VIVS_MC_MMU_RA_PAGE_TABLE, pgtable); } -static const struct etnaviv_iommu_domain_ops etnaviv_iommuv1_ops = { - .free = etnaviv_iommuv1_domain_free, + +const struct etnaviv_iommu_ops etnaviv_iommuv1_ops = { + .free = etnaviv_iommuv1_free, .map = etnaviv_iommuv1_map, .unmap = etnaviv_iommuv1_unmap, .dump_size = etnaviv_iommuv1_dump_size, .dump = etnaviv_iommuv1_dump, + .restore = etnaviv_iommuv1_restore, }; -struct etnaviv_iommu_domain * -etnaviv_iommuv1_domain_alloc(struct etnaviv_gpu *gpu) +struct etnaviv_iommu_context * +etnaviv_iommuv1_context_alloc(struct etnaviv_iommu_global *global) { - struct etnaviv_iommuv1_domain *etnaviv_domain; - struct etnaviv_iommu_domain *domain; - int ret; + struct etnaviv_iommuv1_context *v1_context; + struct etnaviv_iommu_context *context; + + mutex_lock(&global->lock); + + /* + * MMUv1 does not support switching between different contexts without + * a stop the world operation, so we only support a single shared + * context with this version. + */ + if (global->v1.shared_context) { + context = global->v1.shared_context; + etnaviv_iommu_context_get(context); + mutex_unlock(&global->lock); + return context; + } - etnaviv_domain = kzalloc(sizeof(*etnaviv_domain), GFP_KERNEL); - if (!etnaviv_domain) + v1_context = kzalloc(sizeof(*v1_context), GFP_KERNEL); + if (!v1_context) return NULL; - domain = &etnaviv_domain->base; + v1_context->pgtable_cpu = dma_alloc_wc(global->dev, PT_SIZE, + &v1_context->pgtable_dma, + GFP_KERNEL); + if (!v1_context->pgtable_cpu) + goto out_free; - domain->dev = gpu->dev; - domain->base = GPU_MEM_START; - domain->size = PT_ENTRIES * SZ_4K; - domain->ops = &etnaviv_iommuv1_ops; + memset32(v1_context->pgtable_cpu, global->bad_page_dma, PT_ENTRIES); - ret = __etnaviv_iommu_init(etnaviv_domain); - if (ret) - goto out_free; + context = &v1_context->base; + context->global = global; + kref_init(&context->refcount); + mutex_init(&context->lock); + INIT_LIST_HEAD(&context->mappings); + drm_mm_init(&context->mm, GPU_MEM_START, PT_ENTRIES * SZ_4K); + context->global->v1.shared_context = context; + + mutex_unlock(&global->lock); - return &etnaviv_domain->base; + return context; out_free: - kfree(etnaviv_domain); + mutex_unlock(&global->lock); + kfree(v1_context); return NULL; } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu.h b/drivers/gpu/drm/etnaviv/etnaviv_iommu.h deleted file mode 100644 index b279404ce91a..000000000000 --- a/drivers/gpu/drm/etnaviv/etnaviv_iommu.h +++ /dev/null @@ -1,20 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2014-2018 Etnaviv Project - */ - -#ifndef __ETNAVIV_IOMMU_H__ -#define __ETNAVIV_IOMMU_H__ - -struct etnaviv_gpu; -struct etnaviv_iommu_domain; - -struct etnaviv_iommu_domain * -etnaviv_iommuv1_domain_alloc(struct etnaviv_gpu *gpu); -void etnaviv_iommuv1_restore(struct etnaviv_gpu *gpu); - -struct etnaviv_iommu_domain * -etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu); -void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu); - -#endif /* __ETNAVIV_IOMMU_H__ */ diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c index d7cc184da571..5ca2077c148d 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c @@ -13,7 +13,6 @@ #include "etnaviv_cmdbuf.h" #include "etnaviv_gpu.h" #include "etnaviv_mmu.h" -#include "etnaviv_iommu.h" #include "state.xml.h" #include "state_hi.xml.h" @@ -28,11 +27,9 @@ #define MMUv2_MAX_STLB_ENTRIES 1024 -struct etnaviv_iommuv2_domain { - struct etnaviv_iommu_domain base; - /* P(age) T(able) A(rray) */ - u64 *pta_cpu; - dma_addr_t pta_dma; +struct etnaviv_iommuv2_context { + struct etnaviv_iommu_context base; + unsigned short id; /* M(aster) TLB aka first level pagetable */ u32 *mtlb_cpu; dma_addr_t mtlb_dma; @@ -41,41 +38,62 @@ struct etnaviv_iommuv2_domain { dma_addr_t stlb_dma[MMUv2_MAX_STLB_ENTRIES]; }; -static struct etnaviv_iommuv2_domain * -to_etnaviv_domain(struct etnaviv_iommu_domain *domain) +static struct etnaviv_iommuv2_context * +to_v2_context(struct etnaviv_iommu_context *context) { - return container_of(domain, struct etnaviv_iommuv2_domain, base); + return container_of(context, struct etnaviv_iommuv2_context, base); } +static void etnaviv_iommuv2_free(struct etnaviv_iommu_context *context) +{ + struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); + int i; + + drm_mm_takedown(&context->mm); + + for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { + if (v2_context->stlb_cpu[i]) + dma_free_wc(context->global->dev, SZ_4K, + v2_context->stlb_cpu[i], + v2_context->stlb_dma[i]); + } + + dma_free_wc(context->global->dev, SZ_4K, v2_context->mtlb_cpu, + v2_context->mtlb_dma); + + clear_bit(v2_context->id, context->global->v2.pta_alloc); + + vfree(v2_context); +} static int -etnaviv_iommuv2_ensure_stlb(struct etnaviv_iommuv2_domain *etnaviv_domain, +etnaviv_iommuv2_ensure_stlb(struct etnaviv_iommuv2_context *v2_context, int stlb) { - if (etnaviv_domain->stlb_cpu[stlb]) + if (v2_context->stlb_cpu[stlb]) return 0; - etnaviv_domain->stlb_cpu[stlb] = - dma_alloc_wc(etnaviv_domain->base.dev, SZ_4K, - &etnaviv_domain->stlb_dma[stlb], + v2_context->stlb_cpu[stlb] = + dma_alloc_wc(v2_context->base.global->dev, SZ_4K, + &v2_context->stlb_dma[stlb], GFP_KERNEL); - if (!etnaviv_domain->stlb_cpu[stlb]) + if (!v2_context->stlb_cpu[stlb]) return -ENOMEM; - memset32(etnaviv_domain->stlb_cpu[stlb], MMUv2_PTE_EXCEPTION, + memset32(v2_context->stlb_cpu[stlb], MMUv2_PTE_EXCEPTION, SZ_4K / sizeof(u32)); - etnaviv_domain->mtlb_cpu[stlb] = etnaviv_domain->stlb_dma[stlb] | - MMUv2_PTE_PRESENT; + v2_context->mtlb_cpu[stlb] = + v2_context->stlb_dma[stlb] | MMUv2_PTE_PRESENT; + return 0; } -static int etnaviv_iommuv2_map(struct etnaviv_iommu_domain *domain, +static int etnaviv_iommuv2_map(struct etnaviv_iommu_context *context, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { - struct etnaviv_iommuv2_domain *etnaviv_domain = - to_etnaviv_domain(domain); + struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); int mtlb_entry, stlb_entry, ret; u32 entry = lower_32_bits(paddr) | MMUv2_PTE_PRESENT; @@ -91,20 +109,19 @@ static int etnaviv_iommuv2_map(struct etnaviv_iommu_domain *domain, mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT; stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT; - ret = etnaviv_iommuv2_ensure_stlb(etnaviv_domain, mtlb_entry); + ret = etnaviv_iommuv2_ensure_stlb(v2_context, mtlb_entry); if (ret) return ret; - etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = entry; + v2_context->stlb_cpu[mtlb_entry][stlb_entry] = entry; return 0; } -static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_domain *domain, +static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_context *context, unsigned long iova, size_t size) { - struct etnaviv_iommuv2_domain *etnaviv_domain = - to_etnaviv_domain(domain); + struct etnaviv_iommuv2_context *etnaviv_domain = to_v2_context(context); int mtlb_entry, stlb_entry; if (size != SZ_4K) @@ -118,118 +135,35 @@ static size_t etnaviv_iommuv2_unmap(struct etnaviv_iommu_domain *domain, return SZ_4K; } -static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain) -{ - int ret; - - /* allocate scratch page */ - etnaviv_domain->base.bad_page_cpu = - dma_alloc_wc(etnaviv_domain->base.dev, SZ_4K, - &etnaviv_domain->base.bad_page_dma, - GFP_KERNEL); - if (!etnaviv_domain->base.bad_page_cpu) { - ret = -ENOMEM; - goto fail_mem; - } - - memset32(etnaviv_domain->base.bad_page_cpu, 0xdead55aa, - SZ_4K / sizeof(u32)); - - etnaviv_domain->pta_cpu = dma_alloc_wc(etnaviv_domain->base.dev, - SZ_4K, &etnaviv_domain->pta_dma, - GFP_KERNEL); - if (!etnaviv_domain->pta_cpu) { - ret = -ENOMEM; - goto fail_mem; - } - - etnaviv_domain->mtlb_cpu = dma_alloc_wc(etnaviv_domain->base.dev, - SZ_4K, &etnaviv_domain->mtlb_dma, - GFP_KERNEL); - if (!etnaviv_domain->mtlb_cpu) { - ret = -ENOMEM; - goto fail_mem; - } - - memset32(etnaviv_domain->mtlb_cpu, MMUv2_PTE_EXCEPTION, - MMUv2_MAX_STLB_ENTRIES); - - return 0; - -fail_mem: - if (etnaviv_domain->base.bad_page_cpu) - dma_free_wc(etnaviv_domain->base.dev, SZ_4K, - etnaviv_domain->base.bad_page_cpu, - etnaviv_domain->base.bad_page_dma); - - if (etnaviv_domain->pta_cpu) - dma_free_wc(etnaviv_domain->base.dev, SZ_4K, - etnaviv_domain->pta_cpu, etnaviv_domain->pta_dma); - - if (etnaviv_domain->mtlb_cpu) - dma_free_wc(etnaviv_domain->base.dev, SZ_4K, - etnaviv_domain->mtlb_cpu, etnaviv_domain->mtlb_dma); - - return ret; -} - -static void etnaviv_iommuv2_domain_free(struct etnaviv_iommu_domain *domain) -{ - struct etnaviv_iommuv2_domain *etnaviv_domain = - to_etnaviv_domain(domain); - int i; - - dma_free_wc(etnaviv_domain->base.dev, SZ_4K, - etnaviv_domain->base.bad_page_cpu, - etnaviv_domain->base.bad_page_dma); - - dma_free_wc(etnaviv_domain->base.dev, SZ_4K, - etnaviv_domain->pta_cpu, etnaviv_domain->pta_dma); - - dma_free_wc(etnaviv_domain->base.dev, SZ_4K, - etnaviv_domain->mtlb_cpu, etnaviv_domain->mtlb_dma); - - for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) { - if (etnaviv_domain->stlb_cpu[i]) - dma_free_wc(etnaviv_domain->base.dev, SZ_4K, - etnaviv_domain->stlb_cpu[i], - etnaviv_domain->stlb_dma[i]); - } - - vfree(etnaviv_domain); -} - -static size_t etnaviv_iommuv2_dump_size(struct etnaviv_iommu_domain *domain) +static size_t etnaviv_iommuv2_dump_size(struct etnaviv_iommu_context *context) { - struct etnaviv_iommuv2_domain *etnaviv_domain = - to_etnaviv_domain(domain); + struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); size_t dump_size = SZ_4K; int i; for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) - if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT) + if (v2_context->mtlb_cpu[i] & MMUv2_PTE_PRESENT) dump_size += SZ_4K; return dump_size; } -static void etnaviv_iommuv2_dump(struct etnaviv_iommu_domain *domain, void *buf) +static void etnaviv_iommuv2_dump(struct etnaviv_iommu_context *context, void *buf) { - struct etnaviv_iommuv2_domain *etnaviv_domain = - to_etnaviv_domain(domain); + struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); int i; - memcpy(buf, etnaviv_domain->mtlb_cpu, SZ_4K); + memcpy(buf, v2_context->mtlb_cpu, SZ_4K); buf += SZ_4K; for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++, buf += SZ_4K) - if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT) - memcpy(buf, etnaviv_domain->stlb_cpu[i], SZ_4K); + if (v2_context->mtlb_cpu[i] & MMUv2_PTE_PRESENT) + memcpy(buf, v2_context->stlb_cpu[i], SZ_4K); } -static void etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu *gpu) +static void etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu *gpu, + struct etnaviv_iommu_context *context) { - struct etnaviv_iommuv2_domain *etnaviv_domain = - to_etnaviv_domain(gpu->mmu->domain); + struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); u16 prefetch; /* If the MMU is already enabled the state is still there. */ @@ -237,8 +171,8 @@ static void etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu *gpu) return; prefetch = etnaviv_buffer_config_mmuv2(gpu, - (u32)etnaviv_domain->mtlb_dma, - (u32)etnaviv_domain->base.bad_page_dma); + (u32)v2_context->mtlb_dma, + (u32)context->global->bad_page_dma); etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer), prefetch); etnaviv_gpu_wait_idle(gpu, 100); @@ -246,10 +180,10 @@ static void etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu *gpu) gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE); } -static void etnaviv_iommuv2_restore_sec(struct etnaviv_gpu *gpu) +static void etnaviv_iommuv2_restore_sec(struct etnaviv_gpu *gpu, + struct etnaviv_iommu_context *context) { - struct etnaviv_iommuv2_domain *etnaviv_domain = - to_etnaviv_domain(gpu->mmu->domain); + struct etnaviv_iommuv2_context *v2_context = to_v2_context(context); u16 prefetch; /* If the MMU is already enabled the state is still there. */ @@ -257,26 +191,26 @@ static void etnaviv_iommuv2_restore_sec(struct etnaviv_gpu *gpu) return; gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_LOW, - lower_32_bits(etnaviv_domain->pta_dma)); + lower_32_bits(context->global->v2.pta_dma)); gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_HIGH, - upper_32_bits(etnaviv_domain->pta_dma)); + upper_32_bits(context->global->v2.pta_dma)); gpu_write(gpu, VIVS_MMUv2_PTA_CONTROL, VIVS_MMUv2_PTA_CONTROL_ENABLE); gpu_write(gpu, VIVS_MMUv2_NONSEC_SAFE_ADDR_LOW, - lower_32_bits(etnaviv_domain->base.bad_page_dma)); + lower_32_bits(context->global->bad_page_dma)); gpu_write(gpu, VIVS_MMUv2_SEC_SAFE_ADDR_LOW, - lower_32_bits(etnaviv_domain->base.bad_page_dma)); + lower_32_bits(context->global->bad_page_dma)); gpu_write(gpu, VIVS_MMUv2_SAFE_ADDRESS_CONFIG, VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH( - upper_32_bits(etnaviv_domain->base.bad_page_dma)) | + upper_32_bits(context->global->bad_page_dma)) | VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH( - upper_32_bits(etnaviv_domain->base.bad_page_dma))); + upper_32_bits(context->global->bad_page_dma))); - etnaviv_domain->pta_cpu[0] = etnaviv_domain->mtlb_dma | - VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K; + context->global->v2.pta_cpu[0] = v2_context->mtlb_dma | + VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K; /* trigger a PTA load through the FE */ - prefetch = etnaviv_buffer_config_pta(gpu); + prefetch = etnaviv_buffer_config_pta(gpu, v2_context->id); etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer), prefetch); etnaviv_gpu_wait_idle(gpu, 100); @@ -284,14 +218,15 @@ static void etnaviv_iommuv2_restore_sec(struct etnaviv_gpu *gpu) gpu_write(gpu, VIVS_MMUv2_SEC_CONTROL, VIVS_MMUv2_SEC_CONTROL_ENABLE); } -void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu) +static void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu, + struct etnaviv_iommu_context *context) { switch (gpu->sec_mode) { case ETNA_SEC_NONE: - etnaviv_iommuv2_restore_nonsec(gpu); + etnaviv_iommuv2_restore_nonsec(gpu, context); break; case ETNA_SEC_KERNEL: - etnaviv_iommuv2_restore_sec(gpu); + etnaviv_iommuv2_restore_sec(gpu, context); break; default: WARN(1, "unhandled GPU security mode\n"); @@ -299,39 +234,56 @@ void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu) } } -static const struct etnaviv_iommu_domain_ops etnaviv_iommuv2_ops = { - .free = etnaviv_iommuv2_domain_free, +const struct etnaviv_iommu_ops etnaviv_iommuv2_ops = { + .free = etnaviv_iommuv2_free, .map = etnaviv_iommuv2_map, .unmap = etnaviv_iommuv2_unmap, .dump_size = etnaviv_iommuv2_dump_size, .dump = etnaviv_iommuv2_dump, + .restore = etnaviv_iommuv2_restore, }; -struct etnaviv_iommu_domain * -etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu) +struct etnaviv_iommu_context * +etnaviv_iommuv2_context_alloc(struct etnaviv_iommu_global *global) { - struct etnaviv_iommuv2_domain *etnaviv_domain; - struct etnaviv_iommu_domain *domain; - int ret; + struct etnaviv_iommuv2_context *v2_context; + struct etnaviv_iommu_context *context; - etnaviv_domain = vzalloc(sizeof(*etnaviv_domain)); - if (!etnaviv_domain) + v2_context = vzalloc(sizeof(*v2_context)); + if (!v2_context) return NULL; - domain = &etnaviv_domain->base; + mutex_lock(&global->lock); + v2_context->id = find_first_zero_bit(global->v2.pta_alloc, + ETNAVIV_PTA_ENTRIES); + if (v2_context->id < ETNAVIV_PTA_ENTRIES) { + set_bit(v2_context->id, global->v2.pta_alloc); + } else { + mutex_unlock(&global->lock); + goto out_free; + } + mutex_unlock(&global->lock); - domain->dev = gpu->dev; - domain->base = SZ_4K; - domain->size = (u64)SZ_1G * 4 - SZ_4K; - domain->ops = &etnaviv_iommuv2_ops; + v2_context->mtlb_cpu = dma_alloc_wc(global->dev, SZ_4K, + &v2_context->mtlb_dma, GFP_KERNEL); + if (!v2_context->mtlb_cpu) + goto out_free_id; - ret = etnaviv_iommuv2_init(etnaviv_domain); - if (ret) - goto out_free; + memset32(v2_context->mtlb_cpu, MMUv2_PTE_EXCEPTION, + MMUv2_MAX_STLB_ENTRIES); + + context = &v2_context->base; + context->global = global; + kref_init(&context->refcount); + mutex_init(&context->lock); + INIT_LIST_HEAD(&context->mappings); + drm_mm_init(&context->mm, SZ_4K, (u64)SZ_1G * 4 - SZ_4K); - return &etnaviv_domain->base; + return context; +out_free_id: + clear_bit(v2_context->id, global->v2.pta_alloc); out_free: - vfree(etnaviv_domain); + vfree(v2_context); return NULL; } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c index bbd1624a3df8..2f64eef773ed 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c @@ -3,6 +3,7 @@ * Copyright (C) 2015-2018 Etnaviv Project */ +#include <linux/dma-mapping.h> #include <linux/scatterlist.h> #include "common.xml.h" @@ -10,10 +11,9 @@ #include "etnaviv_drv.h" #include "etnaviv_gem.h" #include "etnaviv_gpu.h" -#include "etnaviv_iommu.h" #include "etnaviv_mmu.h" -static void etnaviv_domain_unmap(struct etnaviv_iommu_domain *domain, +static void etnaviv_context_unmap(struct etnaviv_iommu_context *context, unsigned long iova, size_t size) { size_t unmapped_page, unmapped = 0; @@ -26,7 +26,8 @@ static void etnaviv_domain_unmap(struct etnaviv_iommu_domain *domain, } while (unmapped < size) { - unmapped_page = domain->ops->unmap(domain, iova, pgsize); + unmapped_page = context->global->ops->unmap(context, iova, + pgsize); if (!unmapped_page) break; @@ -35,7 +36,7 @@ static void etnaviv_domain_unmap(struct etnaviv_iommu_domain *domain, } } -static int etnaviv_domain_map(struct etnaviv_iommu_domain *domain, +static int etnaviv_context_map(struct etnaviv_iommu_context *context, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { @@ -51,7 +52,8 @@ static int etnaviv_domain_map(struct etnaviv_iommu_domain *domain, } while (size) { - ret = domain->ops->map(domain, iova, paddr, pgsize, prot); + ret = context->global->ops->map(context, iova, paddr, pgsize, + prot); if (ret) break; @@ -62,21 +64,19 @@ static int etnaviv_domain_map(struct etnaviv_iommu_domain *domain, /* unroll mapping in case something went wrong */ if (ret) - etnaviv_domain_unmap(domain, orig_iova, orig_size - size); + etnaviv_context_unmap(context, orig_iova, orig_size - size); return ret; } -static int etnaviv_iommu_map(struct etnaviv_iommu *iommu, u32 iova, +static int etnaviv_iommu_map(struct etnaviv_iommu_context *context, u32 iova, struct sg_table *sgt, unsigned len, int prot) -{ - struct etnaviv_iommu_domain *domain = iommu->domain; - struct scatterlist *sg; +{ struct scatterlist *sg; unsigned int da = iova; unsigned int i, j; int ret; - if (!domain || !sgt) + if (!context || !sgt) return -EINVAL; for_each_sg(sgt->sgl, sg, sgt->nents, i) { @@ -85,7 +85,7 @@ static int etnaviv_iommu_map(struct etnaviv_iommu *iommu, u32 iova, VERB("map[%d]: %08x %08x(%zx)", i, iova, pa, bytes); - ret = etnaviv_domain_map(domain, da, pa, bytes, prot); + ret = etnaviv_context_map(context, da, pa, bytes, prot); if (ret) goto fail; @@ -100,16 +100,15 @@ fail: for_each_sg(sgt->sgl, sg, i, j) { size_t bytes = sg_dma_len(sg) + sg->offset; - etnaviv_domain_unmap(domain, da, bytes); + etnaviv_context_unmap(context, da, bytes); da += bytes; } return ret; } -static void etnaviv_iommu_unmap(struct etnaviv_iommu *iommu, u32 iova, +static void etnaviv_iommu_unmap(struct etnaviv_iommu_context *context, u32 iova, struct sg_table *sgt, unsigned len) { - struct etnaviv_iommu_domain *domain = iommu->domain; struct scatterlist *sg; unsigned int da = iova; int i; @@ -117,7 +116,7 @@ static void etnaviv_iommu_unmap(struct etnaviv_iommu *iommu, u32 iova, for_each_sg(sgt->sgl, sg, sgt->nents, i) { size_t bytes = sg_dma_len(sg) + sg->offset; - etnaviv_domain_unmap(domain, da, bytes); + etnaviv_context_unmap(context, da, bytes); VERB("unmap[%d]: %08x(%zx)", i, iova, bytes); @@ -127,24 +126,24 @@ static void etnaviv_iommu_unmap(struct etnaviv_iommu *iommu, u32 iova, } } -static void etnaviv_iommu_remove_mapping(struct etnaviv_iommu *mmu, +static void etnaviv_iommu_remove_mapping(struct etnaviv_iommu_context *context, struct etnaviv_vram_mapping *mapping) { struct etnaviv_gem_object *etnaviv_obj = mapping->object; - etnaviv_iommu_unmap(mmu, mapping->vram_node.start, + etnaviv_iommu_unmap(context, mapping->vram_node.start, etnaviv_obj->sgt, etnaviv_obj->base.size); drm_mm_remove_node(&mapping->vram_node); } -static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu, +static int etnaviv_iommu_find_iova(struct etnaviv_iommu_context *context, struct drm_mm_node *node, size_t size) { struct etnaviv_vram_mapping *free = NULL; enum drm_mm_insert_mode mode = DRM_MM_INSERT_LOW; int ret; - lockdep_assert_held(&mmu->lock); + lockdep_assert_held(&context->lock); while (1) { struct etnaviv_vram_mapping *m, *n; @@ -152,17 +151,17 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu, struct list_head list; bool found; - ret = drm_mm_insert_node_in_range(&mmu->mm, node, + ret = drm_mm_insert_node_in_range(&context->mm, node, size, 0, 0, 0, U64_MAX, mode); if (ret != -ENOSPC) break; /* Try to retire some entries */ - drm_mm_scan_init(&scan, &mmu->mm, size, 0, 0, mode); + drm_mm_scan_init(&scan, &context->mm, size, 0, 0, mode); found = 0; INIT_LIST_HEAD(&list); - list_for_each_entry(free, &mmu->mappings, mmu_node) { + list_for_each_entry(free, &context->mappings, mmu_node) { /* If this vram node has not been used, skip this. */ if (!free->vram_node.mm) continue; @@ -204,8 +203,8 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu, * this mapping. */ list_for_each_entry_safe(m, n, &list, scan_node) { - etnaviv_iommu_remove_mapping(mmu, m); - m->mmu = NULL; + etnaviv_iommu_remove_mapping(context, m); + m->context = NULL; list_del_init(&m->mmu_node); list_del_init(&m->scan_node); } @@ -221,7 +220,7 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu *mmu, return ret; } -int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, +int etnaviv_iommu_map_gem(struct etnaviv_iommu_context *context, struct etnaviv_gem_object *etnaviv_obj, u32 memory_base, struct etnaviv_vram_mapping *mapping) { @@ -231,17 +230,17 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, lockdep_assert_held(&etnaviv_obj->lock); - mutex_lock(&mmu->lock); + mutex_lock(&context->lock); /* v1 MMU can optimize single entry (contiguous) scatterlists */ - if (mmu->version == ETNAVIV_IOMMU_V1 && + if (context->global->version == ETNAVIV_IOMMU_V1 && sgt->nents == 1 && !(etnaviv_obj->flags & ETNA_BO_FORCE_MMU)) { u32 iova; iova = sg_dma_address(sgt->sgl) - memory_base; if (iova < 0x80000000 - sg_dma_len(sgt->sgl)) { mapping->iova = iova; - list_add_tail(&mapping->mmu_node, &mmu->mappings); + list_add_tail(&mapping->mmu_node, &context->mappings); ret = 0; goto unlock; } @@ -249,12 +248,12 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, node = &mapping->vram_node; - ret = etnaviv_iommu_find_iova(mmu, node, etnaviv_obj->base.size); + ret = etnaviv_iommu_find_iova(context, node, etnaviv_obj->base.size); if (ret < 0) goto unlock; mapping->iova = node->start; - ret = etnaviv_iommu_map(mmu, node->start, sgt, etnaviv_obj->base.size, + ret = etnaviv_iommu_map(context, node->start, sgt, etnaviv_obj->base.size, ETNAVIV_PROT_READ | ETNAVIV_PROT_WRITE); if (ret < 0) { @@ -262,84 +261,63 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, goto unlock; } - list_add_tail(&mapping->mmu_node, &mmu->mappings); - mmu->flush_seq++; + list_add_tail(&mapping->mmu_node, &context->mappings); + context->flush_seq++; unlock: - mutex_unlock(&mmu->lock); + mutex_unlock(&context->lock); return ret; } -void etnaviv_iommu_unmap_gem(struct etnaviv_iommu *mmu, +void etnaviv_iommu_unmap_gem(struct etnaviv_iommu_context *context, struct etnaviv_vram_mapping *mapping) { WARN_ON(mapping->use); - mutex_lock(&mmu->lock); + mutex_lock(&context->lock); /* If the vram node is on the mm, unmap and remove the node */ - if (mapping->vram_node.mm == &mmu->mm) - etnaviv_iommu_remove_mapping(mmu, mapping); + if (mapping->vram_node.mm == &context->mm) + etnaviv_iommu_remove_mapping(context, mapping); list_del(&mapping->mmu_node); - mmu->flush_seq++; - mutex_unlock(&mmu->lock); + context->flush_seq++; + mutex_unlock(&context->lock); } -void etnaviv_iommu_destroy(struct etnaviv_iommu *mmu) +static void etnaviv_iommu_context_free(struct kref *kref) { - drm_mm_takedown(&mmu->mm); - mmu->domain->ops->free(mmu->domain); - kfree(mmu); -} + struct etnaviv_iommu_context *context = + container_of(kref, struct etnaviv_iommu_context, refcount); -struct etnaviv_iommu *etnaviv_iommu_new(struct etnaviv_gpu *gpu) + context->global->ops->free(context); +} +void etnaviv_iommu_context_put(struct etnaviv_iommu_context *context) { - enum etnaviv_iommu_version version; - struct etnaviv_iommu *mmu; - - mmu = kzalloc(sizeof(*mmu), GFP_KERNEL); - if (!mmu) - return ERR_PTR(-ENOMEM); - - if (!(gpu->identity.minor_features1 & chipMinorFeatures1_MMU_VERSION)) { - mmu->domain = etnaviv_iommuv1_domain_alloc(gpu); - version = ETNAVIV_IOMMU_V1; - } else { - mmu->domain = etnaviv_iommuv2_domain_alloc(gpu); - version = ETNAVIV_IOMMU_V2; - } - - if (!mmu->domain) { - dev_err(gpu->dev, "Failed to allocate GPU IOMMU domain\n"); - kfree(mmu); - return ERR_PTR(-ENOMEM); - } - - mmu->gpu = gpu; - mmu->version = version; - mutex_init(&mmu->lock); - INIT_LIST_HEAD(&mmu->mappings); - - drm_mm_init(&mmu->mm, mmu->domain->base, mmu->domain->size); - - return mmu; + kref_put(&context->refcount, etnaviv_iommu_context_free); } -void etnaviv_iommu_restore(struct etnaviv_gpu *gpu) +struct etnaviv_iommu_context * +etnaviv_iommu_context_init(struct etnaviv_iommu_global *global) { - if (gpu->mmu->version == ETNAVIV_IOMMU_V1) - etnaviv_iommuv1_restore(gpu); + if (global->version == ETNAVIV_IOMMU_V1) + return etnaviv_iommuv1_context_alloc(global); else - etnaviv_iommuv2_restore(gpu); + return etnaviv_iommuv2_context_alloc(global); +} + +void etnaviv_iommu_restore(struct etnaviv_gpu *gpu, + struct etnaviv_iommu_context *context) +{ + context->global->ops->restore(gpu, context); } -int etnaviv_iommu_get_suballoc_va(struct etnaviv_iommu *mmu, +int etnaviv_iommu_get_suballoc_va(struct etnaviv_iommu_context *context, struct etnaviv_vram_mapping *mapping, u32 memory_base, dma_addr_t paddr, size_t size) { - mutex_lock(&mmu->lock); + mutex_lock(&context->lock); /* * For MMUv1 we don't add the suballoc region to the pagetables, as @@ -347,40 +325,40 @@ int etnaviv_iommu_get_suballoc_va(struct etnaviv_iommu *mmu, * window. Instead we manufacture a mapping to make it look uniform * to the upper layers. */ - if (mmu->version == ETNAVIV_IOMMU_V1) { + if (context->global->version == ETNAVIV_IOMMU_V1) { mapping->iova = paddr - memory_base; } else { struct drm_mm_node *node = &mapping->vram_node; int ret; - ret = etnaviv_iommu_find_iova(mmu, node, size); + ret = etnaviv_iommu_find_iova(context, node, size); if (ret < 0) { - mutex_unlock(&mmu->lock); + mutex_unlock(&context->lock); return ret; } mapping->iova = node->start; - ret = etnaviv_domain_map(mmu->domain, node->start, paddr, size, - ETNAVIV_PROT_READ); + ret = etnaviv_context_map(context, node->start, paddr, size, + ETNAVIV_PROT_READ); if (ret < 0) { drm_mm_remove_node(node); - mutex_unlock(&mmu->lock); + mutex_unlock(&context->lock); return ret; } - mmu->flush_seq++; + context->flush_seq++; } - list_add_tail(&mapping->mmu_node, &mmu->mappings); + list_add_tail(&mapping->mmu_node, &context->mappings); mapping->use = 1; - mutex_unlock(&mmu->lock); + mutex_unlock(&context->lock); return 0; } -void etnaviv_iommu_put_suballoc_va(struct etnaviv_iommu *mmu, +void etnaviv_iommu_put_suballoc_va(struct etnaviv_iommu_context *context, struct etnaviv_vram_mapping *mapping) { struct drm_mm_node *node = &mapping->vram_node; @@ -390,21 +368,104 @@ void etnaviv_iommu_put_suballoc_va(struct etnaviv_iommu *mmu, mapping->use = 0; - if (mmu->version == ETNAVIV_IOMMU_V1) + if (context->global->version == ETNAVIV_IOMMU_V1) return; - mutex_lock(&mmu->lock); - etnaviv_domain_unmap(mmu->domain, node->start, node->size); + mutex_lock(&context->lock); + etnaviv_context_unmap(context, node->start, node->size); drm_mm_remove_node(node); - mutex_unlock(&mmu->lock); + mutex_unlock(&context->lock); +} + +size_t etnaviv_iommu_dump_size(struct etnaviv_iommu_context *context) +{ + return context->global->ops->dump_size(context); +} + +void etnaviv_iommu_dump(struct etnaviv_iommu_context *context, void *buf) +{ + context->global->ops->dump(context, buf); } -size_t etnaviv_iommu_dump_size(struct etnaviv_iommu *iommu) +int etnaviv_iommu_global_init(struct etnaviv_gpu *gpu) { - return iommu->domain->ops->dump_size(iommu->domain); + enum etnaviv_iommu_version version = ETNAVIV_IOMMU_V1; + struct etnaviv_drm_private *priv = gpu->drm->dev_private; + struct etnaviv_iommu_global *global; + struct device *dev = gpu->drm->dev; + + if (gpu->identity.minor_features1 & chipMinorFeatures1_MMU_VERSION) + version = ETNAVIV_IOMMU_V2; + + if (priv->mmu_global) { + if (priv->mmu_global->version != version) { + dev_err(gpu->dev, + "MMU version doesn't match global version\n"); + return -ENXIO; + } + + priv->mmu_global->use++; + return 0; + } + + global = kzalloc(sizeof(*global), GFP_KERNEL); + if (!global) + return -ENOMEM; + + global->bad_page_cpu = dma_alloc_wc(dev, SZ_4K, &global->bad_page_dma, + GFP_KERNEL); + if (!global->bad_page_cpu) + goto free_global; + + memset32(global->bad_page_cpu, 0xdead55aa, SZ_4K / sizeof(u32)); + + if (version == ETNAVIV_IOMMU_V2) { + global->v2.pta_cpu = dma_alloc_wc(dev, ETNAVIV_PTA_SIZE, + &global->v2.pta_dma, GFP_KERNEL); + if (!global->v2.pta_cpu) + goto free_bad_page; + } + + global->dev = dev; + global->version = version; + global->use = 1; + mutex_init(&global->lock); + + if (version == ETNAVIV_IOMMU_V1) + global->ops = &etnaviv_iommuv1_ops; + else + global->ops = &etnaviv_iommuv2_ops; + + priv->mmu_global = global; + + return 0; + +free_bad_page: + dma_free_wc(dev, SZ_4K, global->bad_page_cpu, global->bad_page_dma); +free_global: + kfree(global); + + return -ENOMEM; } -void etnaviv_iommu_dump(struct etnaviv_iommu *iommu, void *buf) +void etnaviv_iommu_global_fini(struct etnaviv_gpu *gpu) { - iommu->domain->ops->dump(iommu->domain, buf); + struct etnaviv_drm_private *priv = gpu->drm->dev_private; + struct etnaviv_iommu_global *global = priv->mmu_global; + + if (--global->use > 0) + return; + + if (global->v2.pta_cpu) + dma_free_wc(global->dev, ETNAVIV_PTA_SIZE, + global->v2.pta_cpu, global->v2.pta_dma); + + if (global->bad_page_cpu) + dma_free_wc(global->dev, SZ_4K, + global->bad_page_cpu, global->bad_page_dma); + + mutex_destroy(&global->lock); + kfree(global); + + priv->mmu_global = NULL; } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.h b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h index 34afe25df9ca..4438d66db6ab 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h @@ -16,33 +16,58 @@ enum etnaviv_iommu_version { struct etnaviv_gpu; struct etnaviv_vram_mapping; -struct etnaviv_iommu_domain; +struct etnaviv_iommu_global; +struct etnaviv_iommu_context; -struct etnaviv_iommu_domain_ops { - void (*free)(struct etnaviv_iommu_domain *); - int (*map)(struct etnaviv_iommu_domain *domain, unsigned long iova, +struct etnaviv_iommu_ops { + struct etnaviv_iommu_context *(*init)(struct etnaviv_iommu_global *); + void (*free)(struct etnaviv_iommu_context *); + int (*map)(struct etnaviv_iommu_context *context, unsigned long iova, phys_addr_t paddr, size_t size, int prot); - size_t (*unmap)(struct etnaviv_iommu_domain *domain, unsigned long iova, + size_t (*unmap)(struct etnaviv_iommu_context *context, unsigned long iova, size_t size); - size_t (*dump_size)(struct etnaviv_iommu_domain *); - void (*dump)(struct etnaviv_iommu_domain *, void *); + size_t (*dump_size)(struct etnaviv_iommu_context *); + void (*dump)(struct etnaviv_iommu_context *, void *); + void (*restore)(struct etnaviv_gpu *, struct etnaviv_iommu_context *); }; -struct etnaviv_iommu_domain { +extern const struct etnaviv_iommu_ops etnaviv_iommuv1_ops; +extern const struct etnaviv_iommu_ops etnaviv_iommuv2_ops; + +#define ETNAVIV_PTA_SIZE SZ_4K +#define ETNAVIV_PTA_ENTRIES (ETNAVIV_PTA_SIZE / sizeof(u64)) + +struct etnaviv_iommu_global { struct device *dev; + enum etnaviv_iommu_version version; + const struct etnaviv_iommu_ops *ops; + unsigned int use; + struct mutex lock; + void *bad_page_cpu; dma_addr_t bad_page_dma; - u64 base; - u64 size; - const struct etnaviv_iommu_domain_ops *ops; + /* + * This union holds members needed by either MMUv1 or MMUv2, which + * can not exist at the same time. + */ + union { + struct { + struct etnaviv_iommu_context *shared_context; + } v1; + struct { + /* P(age) T(able) A(rray) */ + u64 *pta_cpu; + dma_addr_t pta_dma; + struct spinlock pta_lock; + DECLARE_BITMAP(pta_alloc, ETNAVIV_PTA_ENTRIES); + } v2; + }; }; -struct etnaviv_iommu { - struct etnaviv_gpu *gpu; - struct etnaviv_iommu_domain *domain; - - enum etnaviv_iommu_version version; +struct etnaviv_iommu_context { + struct kref refcount; + struct etnaviv_iommu_global *global; /* memory manager for GPU address area */ struct mutex lock; @@ -51,26 +76,40 @@ struct etnaviv_iommu { unsigned int flush_seq; }; +int etnaviv_iommu_global_init(struct etnaviv_gpu *gpu); +void etnaviv_iommu_global_fini(struct etnaviv_gpu *gpu); + struct etnaviv_gem_object; -int etnaviv_iommu_map_gem(struct etnaviv_iommu *mmu, +int etnaviv_iommu_map_gem(struct etnaviv_iommu_context *context, struct etnaviv_gem_object *etnaviv_obj, u32 memory_base, struct etnaviv_vram_mapping *mapping); -void etnaviv_iommu_unmap_gem(struct etnaviv_iommu *mmu, +void etnaviv_iommu_unmap_gem(struct etnaviv_iommu_context *context, struct etnaviv_vram_mapping *mapping); -int etnaviv_iommu_get_suballoc_va(struct etnaviv_iommu *mmu, +int etnaviv_iommu_get_suballoc_va(struct etnaviv_iommu_context *ctx, struct etnaviv_vram_mapping *mapping, u32 memory_base, dma_addr_t paddr, size_t size); -void etnaviv_iommu_put_suballoc_va(struct etnaviv_iommu *mmu, +void etnaviv_iommu_put_suballoc_va(struct etnaviv_iommu_context *ctx, struct etnaviv_vram_mapping *mapping); -size_t etnaviv_iommu_dump_size(struct etnaviv_iommu *iommu); -void etnaviv_iommu_dump(struct etnaviv_iommu *iommu, void *buf); - -struct etnaviv_iommu *etnaviv_iommu_new(struct etnaviv_gpu *gpu); -void etnaviv_iommu_destroy(struct etnaviv_iommu *iommu); -void etnaviv_iommu_restore(struct etnaviv_gpu *gpu); +size_t etnaviv_iommu_dump_size(struct etnaviv_iommu_context *ctx); +void etnaviv_iommu_dump(struct etnaviv_iommu_context *ctx, void *buf); + +struct etnaviv_iommu_context * +etnaviv_iommu_context_init(struct etnaviv_iommu_global *global); +static inline void etnaviv_iommu_context_get(struct etnaviv_iommu_context *ctx) +{ + kref_get(&ctx->refcount); +} +void etnaviv_iommu_context_put(struct etnaviv_iommu_context *ctx); +void etnaviv_iommu_restore(struct etnaviv_gpu *gpu, + struct etnaviv_iommu_context *ctx); + +struct etnaviv_iommu_context * +etnaviv_iommuv1_context_alloc(struct etnaviv_iommu_global *global); +struct etnaviv_iommu_context * +etnaviv_iommuv2_context_alloc(struct etnaviv_iommu_global *global); #endif /* __ETNAVIV_MMU_H__ */ |