// SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright (c) 2023 Imagination Technologies Ltd. */ #include "pvr_mmu.h" #include "pvr_ccb.h" #include "pvr_device.h" #include "pvr_fw.h" #include "pvr_gem.h" #include "pvr_power.h" #include "pvr_rogue_fwif.h" #include "pvr_rogue_mmu_defs.h" #include #include #include #include #include #include #include #define PVR_SHIFT_FROM_SIZE(size_) (__builtin_ctzll(size_)) #define PVR_MASK_FROM_SIZE(size_) (~((size_) - U64_C(1))) /* * The value of the device page size (%PVR_DEVICE_PAGE_SIZE) is currently * pegged to the host page size (%PAGE_SIZE). This chunk of macro goodness both * ensures that the selected host page size corresponds to a valid device page * size and sets up values needed by the MMU code below. */ #if (PVR_DEVICE_PAGE_SIZE == SZ_4K) # define ROGUE_MMUCTRL_PAGE_SIZE_X ROGUE_MMUCTRL_PAGE_SIZE_4KB # define ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT ROGUE_MMUCTRL_PAGE_4KB_RANGE_SHIFT # define ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK ROGUE_MMUCTRL_PAGE_4KB_RANGE_CLRMSK #elif (PVR_DEVICE_PAGE_SIZE == SZ_16K) # define ROGUE_MMUCTRL_PAGE_SIZE_X ROGUE_MMUCTRL_PAGE_SIZE_16KB # define ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT ROGUE_MMUCTRL_PAGE_16KB_RANGE_SHIFT # define ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK ROGUE_MMUCTRL_PAGE_16KB_RANGE_CLRMSK #elif (PVR_DEVICE_PAGE_SIZE == SZ_64K) # define ROGUE_MMUCTRL_PAGE_SIZE_X ROGUE_MMUCTRL_PAGE_SIZE_64KB # define ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT ROGUE_MMUCTRL_PAGE_64KB_RANGE_SHIFT # define ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK ROGUE_MMUCTRL_PAGE_64KB_RANGE_CLRMSK #elif (PVR_DEVICE_PAGE_SIZE == SZ_256K) # define ROGUE_MMUCTRL_PAGE_SIZE_X ROGUE_MMUCTRL_PAGE_SIZE_256KB # define ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT ROGUE_MMUCTRL_PAGE_256KB_RANGE_SHIFT # define ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK ROGUE_MMUCTRL_PAGE_256KB_RANGE_CLRMSK #elif (PVR_DEVICE_PAGE_SIZE == SZ_1M) # define ROGUE_MMUCTRL_PAGE_SIZE_X ROGUE_MMUCTRL_PAGE_SIZE_1MB # define ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT ROGUE_MMUCTRL_PAGE_1MB_RANGE_SHIFT # define ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK ROGUE_MMUCTRL_PAGE_1MB_RANGE_CLRMSK #elif (PVR_DEVICE_PAGE_SIZE == SZ_2M) # define ROGUE_MMUCTRL_PAGE_SIZE_X ROGUE_MMUCTRL_PAGE_SIZE_2MB # define ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT ROGUE_MMUCTRL_PAGE_2MB_RANGE_SHIFT # define ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK ROGUE_MMUCTRL_PAGE_2MB_RANGE_CLRMSK #else # error Unsupported device page size PVR_DEVICE_PAGE_SIZE #endif #define ROGUE_MMUCTRL_ENTRIES_PT_VALUE_X \ (ROGUE_MMUCTRL_ENTRIES_PT_VALUE >> \ (PVR_DEVICE_PAGE_SHIFT - PVR_SHIFT_FROM_SIZE(SZ_4K))) enum pvr_mmu_sync_level { PVR_MMU_SYNC_LEVEL_NONE = -1, PVR_MMU_SYNC_LEVEL_0 = 0, PVR_MMU_SYNC_LEVEL_1 = 1, PVR_MMU_SYNC_LEVEL_2 = 2, }; #define PVR_MMU_SYNC_LEVEL_0_FLAGS (ROGUE_FWIF_MMUCACHEDATA_FLAGS_PT | \ ROGUE_FWIF_MMUCACHEDATA_FLAGS_INTERRUPT | \ ROGUE_FWIF_MMUCACHEDATA_FLAGS_TLB) #define PVR_MMU_SYNC_LEVEL_1_FLAGS (PVR_MMU_SYNC_LEVEL_0_FLAGS | ROGUE_FWIF_MMUCACHEDATA_FLAGS_PD) #define PVR_MMU_SYNC_LEVEL_2_FLAGS (PVR_MMU_SYNC_LEVEL_1_FLAGS | ROGUE_FWIF_MMUCACHEDATA_FLAGS_PC) /** * pvr_mmu_set_flush_flags() - Set MMU cache flush flags for next call to * pvr_mmu_flush_exec(). * @pvr_dev: Target PowerVR device. * @flags: MMU flush flags. Must be one of %PVR_MMU_SYNC_LEVEL_*_FLAGS. * * This function must be called following any possible change to the MMU page * tables. */ static void pvr_mmu_set_flush_flags(struct pvr_device *pvr_dev, u32 flags) { atomic_fetch_or(flags, &pvr_dev->mmu_flush_cache_flags); } /** * pvr_mmu_flush_request_all() - Request flush of all MMU caches when * subsequently calling pvr_mmu_flush_exec(). * @pvr_dev: Target PowerVR device. * * This function must be called following any possible change to the MMU page * tables. */ void pvr_mmu_flush_request_all(struct pvr_device *pvr_dev) { pvr_mmu_set_flush_flags(pvr_dev, PVR_MMU_SYNC_LEVEL_2_FLAGS); } /** * pvr_mmu_flush_exec() - Execute a flush of all MMU caches previously * requested. * @pvr_dev: Target PowerVR device. * @wait: Do not return until the flush is completed. * * This function must be called prior to submitting any new GPU job. The flush * will complete before the jobs are scheduled, so this can be called once after * a series of maps. However, a single unmap should always be immediately * followed by a flush and it should be explicitly waited by setting @wait. * * As a failure to flush the MMU caches could risk memory corruption, if the * flush fails (implying the firmware is not responding) then the GPU device is * marked as lost. * * Returns: * * 0 on success when @wait is true, or * * -%EIO if the device is unavailable, or * * Any error encountered while submitting the flush command via the KCCB. */ int pvr_mmu_flush_exec(struct pvr_device *pvr_dev, bool wait) { struct rogue_fwif_kccb_cmd cmd_mmu_cache = {}; struct rogue_fwif_mmucachedata *cmd_mmu_cache_data = &cmd_mmu_cache.cmd_data.mmu_cache_data; int err = 0; u32 slot; int idx; if (!drm_dev_enter(from_pvr_device(pvr_dev), &idx)) return -EIO; /* Can't flush MMU if the firmware hasn't booted yet. */ if (!pvr_dev->fw_dev.booted) goto err_drm_dev_exit; cmd_mmu_cache_data->cache_flags = atomic_xchg(&pvr_dev->mmu_flush_cache_flags, 0); if (!cmd_mmu_cache_data->cache_flags) goto err_drm_dev_exit; cmd_mmu_cache.cmd_type = ROGUE_FWIF_KCCB_CMD_MMUCACHE; pvr_fw_object_get_fw_addr(pvr_dev->fw_dev.mem.mmucache_sync_obj, &cmd_mmu_cache_data->mmu_cache_sync_fw_addr); cmd_mmu_cache_data->mmu_cache_sync_update_value = 0; err = pvr_kccb_send_cmd(pvr_dev, &cmd_mmu_cache, &slot); if (err) goto err_reset_and_retry; err = pvr_kccb_wait_for_completion(pvr_dev, slot, HZ, NULL); if (err) goto err_reset_and_retry; drm_dev_exit(idx); return 0; err_reset_and_retry: /* * Flush command failure is most likely the result of a firmware lockup. Hard * reset the GPU and retry. */ err = pvr_power_reset(pvr_dev, true); if (err) goto err_drm_dev_exit; /* Device is lost. */ /* Retry sending flush request. */ err = pvr_kccb_send_cmd(pvr_dev, &cmd_mmu_cache, &slot); if (err) { pvr_device_lost(pvr_dev); goto err_drm_dev_exit; } if (wait) { err = pvr_kccb_wait_for_completion(pvr_dev, slot, HZ, NULL); if (err) pvr_device_lost(pvr_dev); } err_drm_dev_exit: drm_dev_exit(idx); return err; } /** * DOC: PowerVR Virtual Memory Handling */ /** * DOC: PowerVR Virtual Memory Handling (constants) * * .. c:macro:: PVR_IDX_INVALID * * Default value for a u16-based index. * * This value cannot be zero, since zero is a valid index value. */ #define PVR_IDX_INVALID ((u16)(-1)) /** * DOC: MMU backing pages */ /** * DOC: MMU backing pages (constants) * * .. c:macro:: PVR_MMU_BACKING_PAGE_SIZE * * Page size of a PowerVR device's integrated MMU. The CPU page size must be * at least as large as this value for the current implementation; this is * checked at compile-time. */ #define PVR_MMU_BACKING_PAGE_SIZE SZ_4K static_assert(PAGE_SIZE >= PVR_MMU_BACKING_PAGE_SIZE); /** * struct pvr_mmu_backing_page - Represents a single page used to back a page * table of any level. * @dma_addr: DMA address of this page. * @host_ptr: CPU address of this page. * @pvr_dev: The PowerVR device to which this page is associated. **For * internal use only.** */ struct pvr_mmu_backing_page { dma_addr_t dma_addr; void *host_ptr; /* private: internal use only */ struct page *raw_page; struct pvr_device *pvr_dev; }; /** * pvr_mmu_backing_page_init() - Initialize a MMU backing page. * @page: Target backing page. * @pvr_dev: Target PowerVR device. * * This function performs three distinct operations: * * 1. Allocate a single page, * 2. Map the page to the CPU, and * 3. Map the page to DMA-space. * * It is expected that @page be zeroed (e.g. from kzalloc()) before calling * this function. * * Return: * * 0 on success, or * * -%ENOMEM if allocation of the backing page or mapping of the backing * page to DMA fails. */ static int pvr_mmu_backing_page_init(struct pvr_mmu_backing_page *page, struct pvr_device *pvr_dev) { struct device *dev = from_pvr_device(pvr_dev)->dev; struct page *raw_page; int err; dma_addr_t dma_addr; void *host_ptr; raw_page = alloc_page(__GFP_ZERO | GFP_KERNEL); if (!raw_page) return -ENOMEM; host_ptr = vmap(&raw_page, 1, VM_MAP, pgprot_writecombine(PAGE_KERNEL)); if (!host_ptr) { err = -ENOMEM; goto err_free_page; } dma_addr = dma_map_page(dev, raw_page, 0, PVR_MMU_BACKING_PAGE_SIZE, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma_addr)) { err = -ENOMEM; goto err_unmap_page; } page->dma_addr = dma_addr; page->host_ptr = host_ptr; page->pvr_dev = pvr_dev; page->raw_page = raw_page; kmemleak_alloc(page->host_ptr, PAGE_SIZE, 1, GFP_KERNEL); return 0; err_unmap_page: vunmap(host_ptr); err_free_page: __free_page(raw_page); return err; } /** * pvr_mmu_backing_page_fini() - Teardown a MMU backing page. * @page: Target backing page. * * This function performs the mirror operations to pvr_mmu_backing_page_init(), * in reverse order: * * 1. Unmap the page from DMA-space, * 2. Unmap the page from the CPU, and * 3. Free the page. * * It also zeros @page. * * It is a no-op to call this function a second (or further) time on any @page. */ static void pvr_mmu_backing_page_fini(struct pvr_mmu_backing_page *page) { struct device *dev; /* Do nothing if no allocation is present. */ if (!page->pvr_dev) return; dev = from_pvr_device(page->pvr_dev)->dev; dma_unmap_page(dev, page->dma_addr, PVR_MMU_BACKING_PAGE_SIZE, DMA_TO_DEVICE); kmemleak_free(page->host_ptr); vunmap(page->host_ptr); __free_page(page->raw_page); memset(page, 0, sizeof(*page)); } /** * pvr_mmu_backing_page_sync() - Flush a MMU backing page from the CPU to the * device. * @page: Target backing page. * @flags: MMU flush flags. Must be one of %PVR_MMU_SYNC_LEVEL_*_FLAGS. * * .. caution:: * * **This is potentially an expensive function call.** Only call * pvr_mmu_backing_page_sync() once you're sure you have no more changes to * make to the backing page in the immediate future. */ static void pvr_mmu_backing_page_sync(struct pvr_mmu_backing_page *page, u32 flags) { struct pvr_device *pvr_dev = page->pvr_dev; struct device *dev; /* * Do nothing if no allocation is present. This may be the case if * we are unmapping pages. */ if (!pvr_dev) return; dev = from_pvr_device(pvr_dev)->dev; dma_sync_single_for_device(dev, page->dma_addr, PVR_MMU_BACKING_PAGE_SIZE, DMA_TO_DEVICE); pvr_mmu_set_flush_flags(pvr_dev, flags); } /** * DOC: Raw page tables */ #define PVR_PAGE_TABLE_TYPEOF_ENTRY(level_) \ typeof_member(struct pvr_page_table_l##level_##_entry_raw, val) #define PVR_PAGE_TABLE_FIELD_GET(level_, name_, field_, entry_) \ (((entry_).val & \ ~ROGUE_MMUCTRL_##name_##_DATA_##field_##_CLRMSK) >> \ ROGUE_MMUCTRL_##name_##_DATA_##field_##_SHIFT) #define PVR_PAGE_TABLE_FIELD_PREP(level_, name_, field_, val_) \ ((((PVR_PAGE_TABLE_TYPEOF_ENTRY(level_))(val_)) \ << ROGUE_MMUCTRL_##name_##_DATA_##field_##_SHIFT) & \ ~ROGUE_MMUCTRL_##name_##_DATA_##field_##_CLRMSK) /** * struct pvr_page_table_l2_entry_raw - A single entry in a level 2 page table. * @val: The raw value of this entry. * * This type is a structure for type-checking purposes. At compile-time, its * size is checked against %ROGUE_MMUCTRL_ENTRY_SIZE_PC_VALUE. * * The value stored in this structure can be decoded using the following bitmap: * * .. flat-table:: * :widths: 1 5 * :stub-columns: 1 * * * - 31..4 * - **Level 1 Page Table Base Address:** Bits 39..12 of the L1 * page table base address, which is 4KiB aligned. * * * - 3..2 * - *(reserved)* * * * - 1 * - **Pending:** When valid bit is not set, indicates that a valid * entry is pending and the MMU should wait for the driver to map * the entry. This is used to support page demand mapping of * memory. * * * - 0 * - **Valid:** Indicates that the entry contains a valid L1 page * table. If the valid bit is not set, then an attempted use of * the page would result in a page fault. */ struct pvr_page_table_l2_entry_raw { u32 val; } __packed; static_assert(sizeof(struct pvr_page_table_l2_entry_raw) * 8 == ROGUE_MMUCTRL_ENTRY_SIZE_PC_VALUE); static bool pvr_page_table_l2_entry_raw_is_valid(struct pvr_page_table_l2_entry_raw entry) { return PVR_PAGE_TABLE_FIELD_GET(2, PC, VALID, entry); } /** * pvr_page_table_l2_entry_raw_set() - Write a valid entry into a raw level 2 * page table. * @entry: Target raw level 2 page table entry. * @child_table_dma_addr: DMA address of the level 1 page table to be * associated with @entry. * * When calling this function, @child_table_dma_addr must be a valid DMA * address and a multiple of %ROGUE_MMUCTRL_PC_DATA_PD_BASE_ALIGNSIZE. */ static void pvr_page_table_l2_entry_raw_set(struct pvr_page_table_l2_entry_raw *entry, dma_addr_t child_table_dma_addr) { child_table_dma_addr >>= ROGUE_MMUCTRL_PC_DATA_PD_BASE_ALIGNSHIFT; WRITE_ONCE(entry->val, PVR_PAGE_TABLE_FIELD_PREP(2, PC, VALID, true) | PVR_PAGE_TABLE_FIELD_PREP(2, PC, ENTRY_PENDING, false) | PVR_PAGE_TABLE_FIELD_PREP(2, PC, PD_BASE, child_table_dma_addr)); } static void pvr_page_table_l2_entry_raw_clear(struct pvr_page_table_l2_entry_raw *entry) { WRITE_ONCE(entry->val, 0); } /** * struct pvr_page_table_l1_entry_raw - A single entry in a level 1 page table. * @val: The raw value of this entry. * * This type is a structure for type-checking purposes. At compile-time, its * size is checked against %ROGUE_MMUCTRL_ENTRY_SIZE_PD_VALUE. * * The value stored in this structure can be decoded using the following bitmap: * * .. flat-table:: * :widths: 1 5 * :stub-columns: 1 * * * - 63..41 * - *(reserved)* * * * - 40 * - **Pending:** When valid bit is not set, indicates that a valid entry * is pending and the MMU should wait for the driver to map the entry. * This is used to support page demand mapping of memory. * * * - 39..5 * - **Level 0 Page Table Base Address:** The way this value is * interpreted depends on the page size. Bits not specified in the * table below (e.g. bits 11..5 for page size 4KiB) should be * considered reserved. * * This table shows the bits used in an L1 page table entry to * represent the Physical Table Base Address for a given Page Size. * Since each L1 page table entry covers 2MiB of address space, the * maximum page size is 2MiB. * * .. flat-table:: * :widths: 1 1 1 1 * :header-rows: 1 * :stub-columns: 1 * * * - Page size * - L0 page table base address bits * - Number of L0 page table entries * - Size of L0 page table * * * - 4KiB * - 39..12 * - 512 * - 4KiB * * * - 16KiB * - 39..10 * - 128 * - 1KiB * * * - 64KiB * - 39..8 * - 32 * - 256B * * * - 256KiB * - 39..6 * - 8 * - 64B * * * - 1MiB * - 39..5 (4 = '0') * - 2 * - 16B * * * - 2MiB * - 39..5 (4..3 = '00') * - 1 * - 8B * * * - 4 * - *(reserved)* * * * - 3..1 * - **Page Size:** Sets the page size, from 4KiB to 2MiB. * * * - 0 * - **Valid:** Indicates that the entry contains a valid L0 page table. * If the valid bit is not set, then an attempted use of the page would * result in a page fault. */ struct pvr_page_table_l1_entry_raw { u64 val; } __packed; static_assert(sizeof(struct pvr_page_table_l1_entry_raw) * 8 == ROGUE_MMUCTRL_ENTRY_SIZE_PD_VALUE); static bool pvr_page_table_l1_entry_raw_is_valid(struct pvr_page_table_l1_entry_raw entry) { return PVR_PAGE_TABLE_FIELD_GET(1, PD, VALID, entry); } /** * pvr_page_table_l1_entry_raw_set() - Write a valid entry into a raw level 1 * page table. * @entry: Target raw level 1 page table entry. * @child_table_dma_addr: DMA address of the level 0 page table to be * associated with @entry. * * When calling this function, @child_table_dma_addr must be a valid DMA * address and a multiple of 4 KiB. */ static void pvr_page_table_l1_entry_raw_set(struct pvr_page_table_l1_entry_raw *entry, dma_addr_t child_table_dma_addr) { WRITE_ONCE(entry->val, PVR_PAGE_TABLE_FIELD_PREP(1, PD, VALID, true) | PVR_PAGE_TABLE_FIELD_PREP(1, PD, ENTRY_PENDING, false) | PVR_PAGE_TABLE_FIELD_PREP(1, PD, PAGE_SIZE, ROGUE_MMUCTRL_PAGE_SIZE_X) | /* * The use of a 4K-specific macro here is correct. It is * a future optimization to allocate sub-host-page-sized * blocks for individual tables, so the condition that any * page table address is aligned to the size of the * largest (a 4KB) table currently holds. */ (child_table_dma_addr & ~ROGUE_MMUCTRL_PT_BASE_4KB_RANGE_CLRMSK)); } static void pvr_page_table_l1_entry_raw_clear(struct pvr_page_table_l1_entry_raw *entry) { WRITE_ONCE(entry->val, 0); } /** * struct pvr_page_table_l0_entry_raw - A single entry in a level 0 page table. * @val: The raw value of this entry. * * This type is a structure for type-checking purposes. At compile-time, its * size is checked against %ROGUE_MMUCTRL_ENTRY_SIZE_PT_VALUE. * * The value stored in this structure can be decoded using the following bitmap: * * .. flat-table:: * :widths: 1 5 * :stub-columns: 1 * * * - 63 * - *(reserved)* * * * - 62 * - **PM/FW Protect:** Indicates a protected region which only the * Parameter Manager (PM) or firmware processor can write to. * * * - 61..40 * - **VP Page (High):** Virtual-physical page used for Parameter Manager * (PM) memory. This field is only used if the additional level of PB * virtualization is enabled. The VP Page field is needed by the PM in * order to correctly reconstitute the free lists after render * completion. This (High) field holds bits 39..18 of the value; the * Low field holds bits 17..12. Bits 11..0 are always zero because the * value is always aligned to the 4KiB page size. * * * - 39..12 * - **Physical Page Address:** The way this value is interpreted depends * on the page size. Bits not specified in the table below (e.g. bits * 20..12 for page size 2MiB) should be considered reserved. * * This table shows the bits used in an L0 page table entry to represent * the Physical Page Address for a given page size (as defined in the * associated L1 page table entry). * * .. flat-table:: * :widths: 1 1 * :header-rows: 1 * :stub-columns: 1 * * * - Page size * - Physical address bits * * * - 4KiB * - 39..12 * * * - 16KiB * - 39..14 * * * - 64KiB * - 39..16 * * * - 256KiB * - 39..18 * * * - 1MiB * - 39..20 * * * - 2MiB * - 39..21 * * * - 11..6 * - **VP Page (Low):** Continuation of VP Page (High). * * * - 5 * - **Pending:** When valid bit is not set, indicates that a valid entry * is pending and the MMU should wait for the driver to map the entry. * This is used to support page demand mapping of memory. * * * - 4 * - **PM Src:** Set on Parameter Manager (PM) allocated page table * entries when indicated by the PM. Note that this bit will only be set * by the PM, not by the device driver. * * * - 3 * - **SLC Bypass Control:** Specifies requests to this page should bypass * the System Level Cache (SLC), if enabled in SLC configuration. * * * - 2 * - **Cache Coherency:** Indicates that the page is coherent (i.e. it * does not require a cache flush between operations on the CPU and the * device). * * * - 1 * - **Read Only:** If set, this bit indicates that the page is read only. * An attempted write to this page would result in a write-protection * fault. * * * - 0 * - **Valid:** Indicates that the entry contains a valid page. If the * valid bit is not set, then an attempted use of the page would result * in a page fault. */ struct pvr_page_table_l0_entry_raw { u64 val; } __packed; static_assert(sizeof(struct pvr_page_table_l0_entry_raw) * 8 == ROGUE_MMUCTRL_ENTRY_SIZE_PT_VALUE); /** * struct pvr_page_flags_raw - The configurable flags from a single entry in a * level 0 page table. * @val: The raw value of these flags. Since these are a strict subset of * &struct pvr_page_table_l0_entry_raw; use that type for our member here. * * The flags stored in this type are: PM/FW Protect; SLC Bypass Control; Cache * Coherency, and Read Only (bits 62, 3, 2 and 1 respectively). * * This type should never be instantiated directly; instead use * pvr_page_flags_raw_create() to ensure only valid bits of @val are set. */ struct pvr_page_flags_raw { struct pvr_page_table_l0_entry_raw val; } __packed; static_assert(sizeof(struct pvr_page_flags_raw) == sizeof(struct pvr_page_table_l0_entry_raw)); static bool pvr_page_table_l0_entry_raw_is_valid(struct pvr_page_table_l0_entry_raw entry) { return PVR_PAGE_TABLE_FIELD_GET(0, PT, VALID, entry); } /** * pvr_page_table_l0_entry_raw_set() - Write a valid entry into a raw level 0 * page table. * @entry: Target raw level 0 page table entry. * @dma_addr: DMA address of the physical page to be associated with @entry. * @flags: Options to be set on @entry. * * When calling this function, @child_table_dma_addr must be a valid DMA * address and a multiple of %PVR_DEVICE_PAGE_SIZE. * * The @flags parameter is directly assigned into @entry. It is the callers * responsibility to ensure that only bits specified in * &struct pvr_page_flags_raw are set in @flags. */ static void pvr_page_table_l0_entry_raw_set(struct pvr_page_table_l0_entry_raw *entry, dma_addr_t dma_addr, struct pvr_page_flags_raw flags) { WRITE_ONCE(entry->val, PVR_PAGE_TABLE_FIELD_PREP(0, PT, VALID, true) | PVR_PAGE_TABLE_FIELD_PREP(0, PT, ENTRY_PENDING, false) | (dma_addr & ~ROGUE_MMUCTRL_PAGE_X_RANGE_CLRMSK) | flags.val.val); } static void pvr_page_table_l0_entry_raw_clear(struct pvr_page_table_l0_entry_raw *entry) { WRITE_ONCE(entry->val, 0); } /** * pvr_page_flags_raw_create() - Initialize the flag bits of a raw level 0 page * table entry. * @read_only: This page is read-only (see: Read Only). * @cache_coherent: This page does not require cache flushes (see: Cache * Coherency). * @slc_bypass: This page bypasses the device cache (see: SLC Bypass Control). * @pm_fw_protect: This page is only for use by the firmware or Parameter * Manager (see PM/FW Protect). * * For more details on the use of these four options, see their respective * entries in the table under &struct pvr_page_table_l0_entry_raw. * * Return: * A new &struct pvr_page_flags_raw instance which can be passed directly to * pvr_page_table_l0_entry_raw_set() or pvr_page_table_l0_insert(). */ static struct pvr_page_flags_raw pvr_page_flags_raw_create(bool read_only, bool cache_coherent, bool slc_bypass, bool pm_fw_protect) { struct pvr_page_flags_raw flags; flags.val.val = PVR_PAGE_TABLE_FIELD_PREP(0, PT, READ_ONLY, read_only) | PVR_PAGE_TABLE_FIELD_PREP(0, PT, CC, cache_coherent) | PVR_PAGE_TABLE_FIELD_PREP(0, PT, SLC_BYPASS_CTRL, slc_bypass) | PVR_PAGE_TABLE_FIELD_PREP(0, PT, PM_META_PROTECT, pm_fw_protect); return flags; } /** * struct pvr_page_table_l2_raw - The raw data of a level 2 page table. * * This type is a structure for type-checking purposes. At compile-time, its * size is checked against %PVR_MMU_BACKING_PAGE_SIZE. */ struct pvr_page_table_l2_raw { /** @entries: The raw values of this table. */ struct pvr_page_table_l2_entry_raw entries[ROGUE_MMUCTRL_ENTRIES_PC_VALUE]; } __packed; static_assert(sizeof(struct pvr_page_table_l2_raw) == PVR_MMU_BACKING_PAGE_SIZE); /** * struct pvr_page_table_l1_raw - The raw data of a level 1 page table. * * This type is a structure for type-checking purposes. At compile-time, its * size is checked against %PVR_MMU_BACKING_PAGE_SIZE. */ struct pvr_page_table_l1_raw { /** @entries: The raw values of this table. */ struct pvr_page_table_l1_entry_raw entries[ROGUE_MMUCTRL_ENTRIES_PD_VALUE]; } __packed; static_assert(sizeof(struct pvr_page_table_l1_raw) == PVR_MMU_BACKING_PAGE_SIZE); /** * struct pvr_page_table_l0_raw - The raw data of a level 0 page table. * * This type is a structure for type-checking purposes. At compile-time, its * size is checked against %PVR_MMU_BACKING_PAGE_SIZE. * * .. caution:: * * The size of level 0 page tables is variable depending on the page size * specified in the associated level 1 page table entry. Since the device * page size in use is pegged to the host page size, it cannot vary at * runtime. This structure is therefore only defined to contain the required * number of entries for the current device page size. **You should never * read or write beyond the last supported entry.** */ struct pvr_page_table_l0_raw { /** @entries: The raw values of this table. */ struct pvr_page_table_l0_entry_raw entries[ROGUE_MMUCTRL_ENTRIES_PT_VALUE_X]; } __packed; static_assert(sizeof(struct pvr_page_table_l0_raw) <= PVR_MMU_BACKING_PAGE_SIZE); /** * DOC: Mirror page tables */ /* * We pre-declare these types because they cross-depend on pointers to each * other. */ struct pvr_page_table_l1; struct pvr_page_table_l0; /** * struct pvr_page_table_l2 - A wrapped level 2 page table. * * To access the raw part of this table, use pvr_page_table_l2_get_raw(). * Alternatively to access a raw entry directly, use * pvr_page_table_l2_get_entry_raw(). * * A level 2 page table forms the root of the page table tree structure, so * this type has no &parent or &parent_idx members. */ struct pvr_page_table_l2 { /** * @entries: The children of this node in the page table tree * structure. These are also mirror tables. The indexing of this array * is identical to that of the raw equivalent * (&pvr_page_table_l1_raw.entries). */ struct pvr_page_table_l1 *entries[ROGUE_MMUCTRL_ENTRIES_PC_VALUE]; /** * @backing_page: A handle to the memory which holds the raw * equivalent of this table. **For internal use only.** */ struct pvr_mmu_backing_page backing_page; /** * @entry_count: The current number of valid entries (that we know of) * in this table. This value is essentially a refcount - the table is * destroyed when this value is decremented to zero by * pvr_page_table_l2_remove(). */ u16 entry_count; }; /** * pvr_page_table_l2_init() - Initialize a level 2 page table. * @table: Target level 2 page table. * @pvr_dev: Target PowerVR device * * It is expected that @table be zeroed (e.g. from kzalloc()) before calling * this function. * * Return: * * 0 on success, or * * Any error encountered while intializing &table->backing_page using * pvr_mmu_backing_page_init(). */ static int pvr_page_table_l2_init(struct pvr_page_table_l2 *table, struct pvr_device *pvr_dev) { return pvr_mmu_backing_page_init(&table->backing_page, pvr_dev); } /** * pvr_page_table_l2_fini() - Teardown a level 2 page table. * @table: Target level 2 page table. * * It is an error to attempt to use @table after calling this function. */ static void pvr_page_table_l2_fini(struct pvr_page_table_l2 *table) { pvr_mmu_backing_page_fini(&table->backing_page); } /** * pvr_page_table_l2_sync() - Flush a level 2 page table from the CPU to the * device. * @table: Target level 2 page table. * * This is just a thin wrapper around pvr_mmu_backing_page_sync(), so the * warning there applies here too: **Only call pvr_page_table_l2_sync() once * you're sure you have no more changes to make to** @table **in the immediate * future.** * * If child level 1 page tables of @table also need to be flushed, this should * be done first using pvr_page_table_l1_sync() *before* calling this function. */ static void pvr_page_table_l2_sync(struct pvr_page_table_l2 *table) { pvr_mmu_backing_page_sync(&table->backing_page, PVR_MMU_SYNC_LEVEL_2_FLAGS); } /** * pvr_page_table_l2_get_raw() - Access the raw equivalent of a mirror level 2 * page table. * @table: Target level 2 page table. * * Essentially returns the CPU address of the raw equivalent of @table, cast to * a &struct pvr_page_table_l2_raw pointer. * * You probably want to call pvr_page_table_l2_get_entry_raw() instead. * * Return: * The raw equivalent of @table. */ static struct pvr_page_table_l2_raw * pvr_page_table_l2_get_raw(struct pvr_page_table_l2 *table) { return table->backing_page.host_ptr; } /** * pvr_page_table_l2_get_entry_raw() - Access an entry from the raw equivalent * of a mirror level 2 page table. * @table: Target level 2 page table. * @idx: Index of the entry to access. * * Technically this function returns a pointer to a slot in a raw level 2 page * table, since the returned "entry" is not guaranteed to be valid. The caller * must verify the validity of the entry at the returned address (perhaps using * pvr_page_table_l2_entry_raw_is_valid()) before reading or overwriting it. * * The value of @idx is not checked here; it is the callers responsibility to * ensure @idx refers to a valid index within @table before dereferencing the * returned pointer. * * Return: * A pointer to the requested raw level 2 page table entry. */ static struct pvr_page_table_l2_entry_raw * pvr_page_table_l2_get_entry_raw(struct pvr_page_table_l2 *table, u16 idx) { return &pvr_page_table_l2_get_raw(table)->entries[idx]; } /** * pvr_page_table_l2_entry_is_valid() - Check if a level 2 page table entry is * marked as valid. * @table: Target level 2 page table. * @idx: Index of the entry to check. * * The value of @idx is not checked here; it is the callers responsibility to * ensure @idx refers to a valid index within @table before calling this * function. */ static bool pvr_page_table_l2_entry_is_valid(struct pvr_page_table_l2 *table, u16 idx) { struct pvr_page_table_l2_entry_raw entry_raw = *pvr_page_table_l2_get_entry_raw(table, idx); return pvr_page_table_l2_entry_raw_is_valid(entry_raw); } /** * struct pvr_page_table_l1 - A wrapped level 1 page table. * * To access the raw part of this table, use pvr_page_table_l1_get_raw(). * Alternatively to access a raw entry directly, use * pvr_page_table_l1_get_entry_raw(). */ struct pvr_page_table_l1 { /** * @entries: The children of this node in the page table tree * structure. These are also mirror tables. The indexing of this array * is identical to that of the raw equivalent * (&pvr_page_table_l0_raw.entries). */ struct pvr_page_table_l0 *entries[ROGUE_MMUCTRL_ENTRIES_PD_VALUE]; /** * @backing_page: A handle to the memory which holds the raw * equivalent of this table. **For internal use only.** */ struct pvr_mmu_backing_page backing_page; union { /** * @parent: The parent of this node in the page table tree structure. * * This is also a mirror table. * * Only valid when the L1 page table is active. When the L1 page table * has been removed and queued for destruction, the next_free field * should be used instead. */ struct pvr_page_table_l2 *parent; /** * @next_free: Pointer to the next L1 page table to take/free. * * Used to form a linked list of L1 page tables. This is used * when preallocating tables and when the page table has been * removed and queued for destruction. */ struct pvr_page_table_l1 *next_free; }; /** * @parent_idx: The index of the entry in the parent table (see * @parent) which corresponds to this table. */ u16 parent_idx; /** * @entry_count: The current number of valid entries (that we know of) * in this table. This value is essentially a refcount - the table is * destroyed when this value is decremented to zero by * pvr_page_table_l1_remove(). */ u16 entry_count; }; /** * pvr_page_table_l1_init() - Initialize a level 1 page table. * @table: Target level 1 page table. * @pvr_dev: Target PowerVR device * * When this function returns successfully, @table is still not considered * valid. It must be inserted into the page table tree structure with * pvr_page_table_l2_insert() before it is ready for use. * * It is expected that @table be zeroed (e.g. from kzalloc()) before calling * this function. * * Return: * * 0 on success, or * * Any error encountered while intializing &table->backing_page using * pvr_mmu_backing_page_init(). */ static int pvr_page_table_l1_init(struct pvr_page_table_l1 *table, struct pvr_device *pvr_dev) { table->parent_idx = PVR_IDX_INVALID; return pvr_mmu_backing_page_init(&table->backing_page, pvr_dev); } /** * pvr_page_table_l1_free() - Teardown a level 1 page table. * @table: Target level 1 page table. * * It is an error to attempt to use @table after calling this function, even * indirectly. This includes calling pvr_page_table_l2_remove(), which must * be called *before* pvr_page_table_l1_free(). */ static void pvr_page_table_l1_free(struct pvr_page_table_l1 *table) { pvr_mmu_backing_page_fini(&table->backing_page); kfree(table); } /** * pvr_page_table_l1_sync() - Flush a level 1 page table from the CPU to the * device. * @table: Target level 1 page table. * * This is just a thin wrapper around pvr_mmu_backing_page_sync(), so the * warning there applies here too: **Only call pvr_page_table_l1_sync() once * you're sure you have no more changes to make to** @table **in the immediate * future.** * * If child level 0 page tables of @table also need to be flushed, this should * be done first using pvr_page_table_l0_sync() *before* calling this function. */ static void pvr_page_table_l1_sync(struct pvr_page_table_l1 *table) { pvr_mmu_backing_page_sync(&table->backing_page, PVR_MMU_SYNC_LEVEL_1_FLAGS); } /** * pvr_page_table_l1_get_raw() - Access the raw equivalent of a mirror level 1 * page table. * @table: Target level 1 page table. * * Essentially returns the CPU address of the raw equivalent of @table, cast to * a &struct pvr_page_table_l1_raw pointer. * * You probably want to call pvr_page_table_l1_get_entry_raw() instead. * * Return: * The raw equivalent of @table. */ static struct pvr_page_table_l1_raw * pvr_page_table_l1_get_raw(struct pvr_page_table_l1 *table) { return table->backing_page.host_ptr; } /** * pvr_page_table_l1_get_entry_raw() - Access an entry from the raw equivalent * of a mirror level 1 page table. * @table: Target level 1 page table. * @idx: Index of the entry to access. * * Technically this function returns a pointer to a slot in a raw level 1 page * table, since the returned "entry" is not guaranteed to be valid. The caller * must verify the validity of the entry at the returned address (perhaps using * pvr_page_table_l1_entry_raw_is_valid()) before reading or overwriting it. * * The value of @idx is not checked here; it is the callers responsibility to * ensure @idx refers to a valid index within @table before dereferencing the * returned pointer. * * Return: * A pointer to the requested raw level 1 page table entry. */ static struct pvr_page_table_l1_entry_raw * pvr_page_table_l1_get_entry_raw(struct pvr_page_table_l1 *table, u16 idx) { return &pvr_page_table_l1_get_raw(table)->entries[idx]; } /** * pvr_page_table_l1_entry_is_valid() - Check if a level 1 page table entry is * marked as valid. * @table: Target level 1 page table. * @idx: Index of the entry to check. * * The value of @idx is not checked here; it is the callers responsibility to * ensure @idx refers to a valid index within @table before calling this * function. */ static bool pvr_page_table_l1_entry_is_valid(struct pvr_page_table_l1 *table, u16 idx) { struct pvr_page_table_l1_entry_raw entry_raw = *pvr_page_table_l1_get_entry_raw(table, idx); return pvr_page_table_l1_entry_raw_is_valid(entry_raw); } /** * struct pvr_page_table_l0 - A wrapped level 0 page table. * * To access the raw part of this table, use pvr_page_table_l0_get_raw(). * Alternatively to access a raw entry directly, use * pvr_page_table_l0_get_entry_raw(). * * There is no mirror representation of an individual page, so this type has no * &entries member. */ struct pvr_page_table_l0 { /** * @backing_page: A handle to the memory which holds the raw * equivalent of this table. **For internal use only.** */ struct pvr_mmu_backing_page backing_page; union { /** * @parent: The parent of this node in the page table tree structure. * * This is also a mirror table. * * Only valid when the L0 page table is active. When the L0 page table * has been removed and queued for destruction, the next_free field * should be used instead. */ struct pvr_page_table_l1 *parent; /** * @next_free: Pointer to the next L0 page table to take/free. * * Used to form a linked list of L0 page tables. This is used * when preallocating tables and when the page table has been * removed and queued for destruction. */ struct pvr_page_table_l0 *next_free; }; /** * @parent_idx: The index of the entry in the parent table (see * @parent) which corresponds to this table. */ u16 parent_idx; /** * @entry_count: The current number of valid entries (that we know of) * in this table. This value is essentially a refcount - the table is * destroyed when this value is decremented to zero by * pvr_page_table_l0_remove(). */ u16 entry_count; }; /** * pvr_page_table_l0_init() - Initialize a level 0 page table. * @table: Target level 0 page table. * @pvr_dev: Target PowerVR device * * When this function returns successfully, @table is still not considered * valid. It must be inserted into the page table tree structure with * pvr_page_table_l1_insert() before it is ready for use. * * It is expected that @table be zeroed (e.g. from kzalloc()) before calling * this function. * * Return: * * 0 on success, or * * Any error encountered while intializing &table->backing_page using * pvr_mmu_backing_page_init(). */ static int pvr_page_table_l0_init(struct pvr_page_table_l0 *table, struct pvr_device *pvr_dev) { table->parent_idx = PVR_IDX_INVALID; return pvr_mmu_backing_page_init(&table->backing_page, pvr_dev); } /** * pvr_page_table_l0_free() - Teardown a level 0 page table. * @table: Target level 0 page table. * * It is an error to attempt to use @table after calling this function, even * indirectly. This includes calling pvr_page_table_l1_remove(), which must * be called *before* pvr_page_table_l0_free(). */ static void pvr_page_table_l0_free(struct pvr_page_table_l0 *table) { pvr_mmu_backing_page_fini(&table->backing_page); kfree(table); } /** * pvr_page_table_l0_sync() - Flush a level 0 page table from the CPU to the * device. * @table: Target level 0 page table. * * This is just a thin wrapper around pvr_mmu_backing_page_sync(), so the * warning there applies here too: **Only call pvr_page_table_l0_sync() once * you're sure you have no more changes to make to** @table **in the immediate * future.** * * If child pages of @table also need to be flushed, this should be done first * using a DMA sync function (e.g. dma_sync_sg_for_device()) *before* calling * this function. */ static void pvr_page_table_l0_sync(struct pvr_page_table_l0 *table) { pvr_mmu_backing_page_sync(&table->backing_page, PVR_MMU_SYNC_LEVEL_0_FLAGS); } /** * pvr_page_table_l0_get_raw() - Access the raw equivalent of a mirror level 0 * page table. * @table: Target level 0 page table. * * Essentially returns the CPU address of the raw equivalent of @table, cast to * a &struct pvr_page_table_l0_raw pointer. * * You probably want to call pvr_page_table_l0_get_entry_raw() instead. * * Return: * The raw equivalent of @table. */ static struct pvr_page_table_l0_raw * pvr_page_table_l0_get_raw(struct pvr_page_table_l0 *table) { return table->backing_page.host_ptr; } /** * pvr_page_table_l0_get_entry_raw() - Access an entry from the raw equivalent * of a mirror level 0 page table. * @table: Target level 0 page table. * @idx: Index of the entry to access. * * Technically this function returns a pointer to a slot in a raw level 0 page * table, since the returned "entry" is not guaranteed to be valid. The caller * must verify the validity of the entry at the returned address (perhaps using * pvr_page_table_l0_entry_raw_is_valid()) before reading or overwriting it. * * The value of @idx is not checked here; it is the callers responsibility to * ensure @idx refers to a valid index within @table before dereferencing the * returned pointer. This is espcially important for level 0 page tables, which * can have a variable number of entries. * * Return: * A pointer to the requested raw level 0 page table entry. */ static struct pvr_page_table_l0_entry_raw * pvr_page_table_l0_get_entry_raw(struct pvr_page_table_l0 *table, u16 idx) { return &pvr_page_table_l0_get_raw(table)->entries[idx]; } /** * pvr_page_table_l0_entry_is_valid() - Check if a level 0 page table entry is * marked as valid. * @table: Target level 0 page table. * @idx: Index of the entry to check. * * The value of @idx is not checked here; it is the callers responsibility to * ensure @idx refers to a valid index within @table before calling this * function. */ static bool pvr_page_table_l0_entry_is_valid(struct pvr_page_table_l0 *table, u16 idx) { struct pvr_page_table_l0_entry_raw entry_raw = *pvr_page_table_l0_get_entry_raw(table, idx); return pvr_page_table_l0_entry_raw_is_valid(entry_raw); } /** * struct pvr_mmu_context - context holding data for operations at page * catalogue level, intended for use with a VM context. */ struct pvr_mmu_context { /** @pvr_dev: The PVR device associated with the owning VM context. */ struct pvr_device *pvr_dev; /** @page_table_l2: The MMU table root. */ struct pvr_page_table_l2 page_table_l2; }; /** * struct pvr_page_table_ptr - A reference to a single physical page as indexed * by the page table structure. * * Intended for embedding in a &struct pvr_mmu_op_context. */ struct pvr_page_table_ptr { /** * @l1_table: A cached handle to the level 1 page table the * context is currently traversing. */ struct pvr_page_table_l1 *l1_table; /** * @l0_table: A cached handle to the level 0 page table the * context is currently traversing. */ struct pvr_page_table_l0 *l0_table; /** * @l2_idx: Index into the level 2 page table the context is * currently referencing. */ u16 l2_idx; /** * @l1_idx: Index into the level 1 page table the context is * currently referencing. */ u16 l1_idx; /** * @l0_idx: Index into the level 0 page table the context is * currently referencing. */ u16 l0_idx; }; /** * struct pvr_mmu_op_context - context holding data for individual * device-virtual mapping operations. Intended for use with a VM bind operation. */ struct pvr_mmu_op_context { /** @mmu_ctx: The MMU context associated with the owning VM context. */ struct pvr_mmu_context *mmu_ctx; /** @map: Data specifically for map operations. */ struct { /** * @sgt: Scatter gather table containing pages pinned for use by * this context - these are currently pinned when initialising * the VM bind operation. */ struct sg_table *sgt; /** @sgt_offset: Start address of the device-virtual mapping. */ u64 sgt_offset; /** * @l1_prealloc_tables: Preallocated l1 page table objects * use by this context when creating a page mapping. Linked list * fully created during initialisation. */ struct pvr_page_table_l1 *l1_prealloc_tables; /** * @l0_prealloc_tables: Preallocated l0 page table objects * use by this context when creating a page mapping. Linked list * fully created during initialisation. */ struct pvr_page_table_l0 *l0_prealloc_tables; } map; /** @unmap: Data specifically for unmap operations. */ struct { /** * @l1_free_tables: Collects page table objects freed by unmap * ops. Linked list empty at creation. */ struct pvr_page_table_l1 *l1_free_tables; /** * @l0_free_tables: Collects page table objects freed by unmap * ops. Linked list empty at creation. */ struct pvr_page_table_l0 *l0_free_tables; } unmap; /** * @curr_page: A reference to a single physical page as indexed by the * page table structure. */ struct pvr_page_table_ptr curr_page; /** * @sync_level_required: The maximum level of the page table tree * structure which has (possibly) been modified since it was last * flushed to the device. * * This field should only be set with pvr_mmu_op_context_require_sync() * or indirectly by pvr_mmu_op_context_sync_partial(). */ enum pvr_mmu_sync_level sync_level_required; }; /** * pvr_page_table_l2_insert() - Insert an entry referring to a level 1 page * table into a level 2 page table. * @op_ctx: Target MMU op context pointing at the entry to insert the L1 page * table into. * @child_table: Target level 1 page table to be referenced by the new entry. * * It is the caller's responsibility to ensure @op_ctx.curr_page points to a * valid L2 entry. * * It is the caller's responsibility to execute any memory barries to ensure * that the creation of @child_table is ordered before the L2 entry is inserted. */ static void pvr_page_table_l2_insert(struct pvr_mmu_op_context *op_ctx, struct pvr_page_table_l1 *child_table) { struct pvr_page_table_l2 *l2_table = &op_ctx->mmu_ctx->page_table_l2; struct pvr_page_table_l2_entry_raw *entry_raw = pvr_page_table_l2_get_entry_raw(l2_table, op_ctx->curr_page.l2_idx); pvr_page_table_l2_entry_raw_set(entry_raw, child_table->backing_page.dma_addr); child_table->parent = l2_table; child_table->parent_idx = op_ctx->curr_page.l2_idx; l2_table->entries[op_ctx->curr_page.l2_idx] = child_table; ++l2_table->entry_count; op_ctx->curr_page.l1_table = child_table; } /** * pvr_page_table_l2_remove() - Remove a level 1 page table from a level 2 page * table. * @op_ctx: Target MMU op context pointing at the L2 entry to remove. * * It is the caller's responsibility to ensure @op_ctx.curr_page points to a * valid L2 entry. */ static void pvr_page_table_l2_remove(struct pvr_mmu_op_context *op_ctx) { struct pvr_page_table_l2 *l2_table = &op_ctx->mmu_ctx->page_table_l2; struct pvr_page_table_l2_entry_raw *entry_raw = pvr_page_table_l2_get_entry_raw(l2_table, op_ctx->curr_page.l1_table->parent_idx); WARN_ON(op_ctx->curr_page.l1_table->parent != l2_table); pvr_page_table_l2_entry_raw_clear(entry_raw); l2_table->entries[op_ctx->curr_page.l1_table->parent_idx] = NULL; op_ctx->curr_page.l1_table->parent_idx = PVR_IDX_INVALID; op_ctx->curr_page.l1_table->next_free = op_ctx->unmap.l1_free_tables; op_ctx->unmap.l1_free_tables = op_ctx->curr_page.l1_table; op_ctx->curr_page.l1_table = NULL; --l2_table->entry_count; } /** * pvr_page_table_l1_insert() - Insert an entry referring to a level 0 page * table into a level 1 page table. * @op_ctx: Target MMU op context pointing at the entry to insert the L0 page * table into. * @child_table: L0 page table to insert. * * It is the caller's responsibility to ensure @op_ctx.curr_page points to a * valid L1 entry. * * It is the caller's responsibility to execute any memory barries to ensure * that the creation of @child_table is ordered before the L1 entry is inserted. */ static void pvr_page_table_l1_insert(struct pvr_mmu_op_context *op_ctx, struct pvr_page_table_l0 *child_table) { struct pvr_page_table_l1_entry_raw *entry_raw = pvr_page_table_l1_get_entry_raw(op_ctx->curr_page.l1_table, op_ctx->curr_page.l1_idx); pvr_page_table_l1_entry_raw_set(entry_raw, child_table->backing_page.dma_addr); child_table->parent = op_ctx->curr_page.l1_table; child_table->parent_idx = op_ctx->curr_page.l1_idx; op_ctx->curr_page.l1_table->entries[op_ctx->curr_page.l1_idx] = child_table; ++op_ctx->curr_page.l1_table->entry_count; op_ctx->curr_page.l0_table = child_table; } /** * pvr_page_table_l1_remove() - Remove a level 0 page table from a level 1 page * table. * @op_ctx: Target MMU op context pointing at the L1 entry to remove. * * If this function results in the L1 table becoming empty, it will be removed * from its parent level 2 page table and destroyed. * * It is the caller's responsibility to ensure @op_ctx.curr_page points to a * valid L1 entry. */ static void pvr_page_table_l1_remove(struct pvr_mmu_op_context *op_ctx) { struct pvr_page_table_l1_entry_raw *entry_raw = pvr_page_table_l1_get_entry_raw(op_ctx->curr_page.l0_table->parent, op_ctx->curr_page.l0_table->parent_idx); WARN_ON(op_ctx->curr_page.l0_table->parent != op_ctx->curr_page.l1_table); pvr_page_table_l1_entry_raw_clear(entry_raw); op_ctx->curr_page.l1_table->entries[op_ctx->curr_page.l0_table->parent_idx] = NULL; op_ctx->curr_page.l0_table->parent_idx = PVR_IDX_INVALID; op_ctx->curr_page.l0_table->next_free = op_ctx->unmap.l0_free_tables; op_ctx->unmap.l0_free_tables = op_ctx->curr_page.l0_table; op_ctx->curr_page.l0_table = NULL; if (--op_ctx->curr_page.l1_table->entry_count == 0) { /* Clear the parent L2 page table entry. */ if (op_ctx->curr_page.l1_table->parent_idx != PVR_IDX_INVALID) pvr_page_table_l2_remove(op_ctx); } } /** * pvr_page_table_l0_insert() - Insert an entry referring to a physical page * into a level 0 page table. * @op_ctx: Target MMU op context pointing at the L0 entry to insert. * @dma_addr: Target DMA address to be referenced by the new entry. * @flags: Page options to be stored in the new entry. * * It is the caller's responsibility to ensure @op_ctx.curr_page points to a * valid L0 entry. */ static void pvr_page_table_l0_insert(struct pvr_mmu_op_context *op_ctx, dma_addr_t dma_addr, struct pvr_page_flags_raw flags) { struct pvr_page_table_l0_entry_raw *entry_raw = pvr_page_table_l0_get_entry_raw(op_ctx->curr_page.l0_table, op_ctx->curr_page.l0_idx); pvr_page_table_l0_entry_raw_set(entry_raw, dma_addr, flags); /* * There is no entry to set here - we don't keep a mirror of * individual pages. */ ++op_ctx->curr_page.l0_table->entry_count; } /** * pvr_page_table_l0_remove() - Remove a physical page from a level 0 page * table. * @op_ctx: Target MMU op context pointing at the L0 entry to remove. * * If this function results in the L0 table becoming empty, it will be removed * from its parent L1 page table and destroyed. * * It is the caller's responsibility to ensure @op_ctx.curr_page points to a * valid L0 entry. */ static void pvr_page_table_l0_remove(struct pvr_mmu_op_context *op_ctx) { struct pvr_page_table_l0_entry_raw *entry_raw = pvr_page_table_l0_get_entry_raw(op_ctx->curr_page.l0_table, op_ctx->curr_page.l0_idx); pvr_page_table_l0_entry_raw_clear(entry_raw); /* * There is no entry to clear here - we don't keep a mirror of * individual pages. */ if (--op_ctx->curr_page.l0_table->entry_count == 0) { /* Clear the parent L1 page table entry. */ if (op_ctx->curr_page.l0_table->parent_idx != PVR_IDX_INVALID) pvr_page_table_l1_remove(op_ctx); } } /** * DOC: Page table index utilities */ /** * pvr_page_table_l2_idx() - Calculate the level 2 page table index for a * device-virtual address. * @device_addr: Target device-virtual address. * * This function does not perform any bounds checking - it is the caller's * responsibility to ensure that @device_addr is valid before interpreting * the result. * * Return: * The index into a level 2 page table corresponding to @device_addr. */ static u16 pvr_page_table_l2_idx(u64 device_addr) { return (device_addr & ~ROGUE_MMUCTRL_VADDR_PC_INDEX_CLRMSK) >> ROGUE_MMUCTRL_VADDR_PC_INDEX_SHIFT; } /** * pvr_page_table_l1_idx() - Calculate the level 1 page table index for a * device-virtual address. * @device_addr: Target device-virtual address. * * This function does not perform any bounds checking - it is the caller's * responsibility to ensure that @device_addr is valid before interpreting * the result. * * Return: * The index into a level 1 page table corresponding to @device_addr. */ static u16 pvr_page_table_l1_idx(u64 device_addr) { return (device_addr & ~ROGUE_MMUCTRL_VADDR_PD_INDEX_CLRMSK) >> ROGUE_MMUCTRL_VADDR_PD_INDEX_SHIFT; } /** * pvr_page_table_l0_idx() - Calculate the level 0 page table index for a * device-virtual address. * @device_addr: Target device-virtual address. * * This function does not perform any bounds checking - it is the caller's * responsibility to ensure that @device_addr is valid before interpreting * the result. * * Return: * The index into a level 0 page table corresponding to @device_addr. */ static u16 pvr_page_table_l0_idx(u64 device_addr) { return (device_addr & ~ROGUE_MMUCTRL_VADDR_PT_INDEX_CLRMSK) >> ROGUE_MMUCTRL_PAGE_X_RANGE_SHIFT; } /** * DOC: High-level page table operations */ /** * pvr_page_table_l1_get_or_insert() - Retrieves (optionally inserting if * necessary) a level 1 page table from the specified level 2 page table entry. * @op_ctx: Target MMU op context. * @should_insert: [IN] Specifies whether new page tables should be inserted * when empty page table entries are encountered during traversal. * * Return: * * 0 on success, or * * If @should_insert is %false: * * -%ENXIO if a level 1 page table would have been inserted. * * If @should_insert is %true: * * Any error encountered while inserting the level 1 page table. */ static int pvr_page_table_l1_get_or_insert(struct pvr_mmu_op_context *op_ctx, bool should_insert) { struct pvr_page_table_l2 *l2_table = &op_ctx->mmu_ctx->page_table_l2; struct pvr_page_table_l1 *table; if (pvr_page_table_l2_entry_is_valid(l2_table, op_ctx->curr_page.l2_idx)) { op_ctx->curr_page.l1_table = l2_table->entries[op_ctx->curr_page.l2_idx]; return 0; } if (!should_insert) return -ENXIO; /* Take a prealloced table. */ table = op_ctx->map.l1_prealloc_tables; if (!table) return -ENOMEM; /* Pop */ op_ctx->map.l1_prealloc_tables = table->next_free; table->next_free = NULL; /* Ensure new table is fully written out before adding to L2 page table. */ wmb(); pvr_page_table_l2_insert(op_ctx, table); return 0; } /** * pvr_page_table_l0_get_or_insert() - Retrieves (optionally inserting if * necessary) a level 0 page table from the specified level 1 page table entry. * @op_ctx: Target MMU op context. * @should_insert: [IN] Specifies whether new page tables should be inserted * when empty page table entries are encountered during traversal. * * Return: * * 0 on success, * * If @should_insert is %false: * * -%ENXIO if a level 0 page table would have been inserted. * * If @should_insert is %true: * * Any error encountered while inserting the level 0 page table. */ static int pvr_page_table_l0_get_or_insert(struct pvr_mmu_op_context *op_ctx, bool should_insert) { struct pvr_page_table_l0 *table; if (pvr_page_table_l1_entry_is_valid(op_ctx->curr_page.l1_table, op_ctx->curr_page.l1_idx)) { op_ctx->curr_page.l0_table = op_ctx->curr_page.l1_table->entries[op_ctx->curr_page.l1_idx]; return 0; } if (!should_insert) return -ENXIO; /* Take a prealloced table. */ table = op_ctx->map.l0_prealloc_tables; if (!table) return -ENOMEM; /* Pop */ op_ctx->map.l0_prealloc_tables = table->next_free; table->next_free = NULL; /* Ensure new table is fully written out before adding to L1 page table. */ wmb(); pvr_page_table_l1_insert(op_ctx, table); return 0; } /** * pvr_mmu_context_create() - Create an MMU context. * @pvr_dev: PVR device associated with owning VM context. * * Returns: * * Newly created MMU context object on success, or * * -%ENOMEM if no memory is available, * * Any error code returned by pvr_page_table_l2_init(). */ struct pvr_mmu_context *pvr_mmu_context_create(struct pvr_device *pvr_dev) { struct pvr_mmu_context *ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); int err; if (!ctx) return ERR_PTR(-ENOMEM); err = pvr_page_table_l2_init(&ctx->page_table_l2, pvr_dev); if (err) return ERR_PTR(err); ctx->pvr_dev = pvr_dev; return ctx; } /** * pvr_mmu_context_destroy() - Destroy an MMU context. * @ctx: Target MMU context. */ void pvr_mmu_context_destroy(struct pvr_mmu_context *ctx) { pvr_page_table_l2_fini(&ctx->page_table_l2); kfree(ctx); } /** * pvr_mmu_get_root_table_dma_addr() - Get the DMA address of the root of the * page table structure behind a VM context. * @ctx: Target MMU context. */ dma_addr_t pvr_mmu_get_root_table_dma_addr(struct pvr_mmu_context *ctx) { return ctx->page_table_l2.backing_page.dma_addr; } /** * pvr_page_table_l1_alloc() - Allocate a l1 page_table object. * @ctx: MMU context of owning VM context. * * Returns: * * Newly created page table object on success, or * * -%ENOMEM if no memory is available, * * Any error code returned by pvr_page_table_l1_init(). */ static struct pvr_page_table_l1 * pvr_page_table_l1_alloc(struct pvr_mmu_context *ctx) { int err; struct pvr_page_table_l1 *table = kzalloc(sizeof(*table), GFP_KERNEL); if (!table) return ERR_PTR(-ENOMEM); err = pvr_page_table_l1_init(table, ctx->pvr_dev); if (err) { kfree(table); return ERR_PTR(err); } return table; } /** * pvr_page_table_l0_alloc() - Allocate a l0 page_table object. * @ctx: MMU context of owning VM context. * * Returns: * * Newly created page table object on success, or * * -%ENOMEM if no memory is available, * * Any error code returned by pvr_page_table_l0_init(). */ static struct pvr_page_table_l0 * pvr_page_table_l0_alloc(struct pvr_mmu_context *ctx) { int err; struct pvr_page_table_l0 *table = kzalloc(sizeof(*table), GFP_KERNEL); if (!table) return ERR_PTR(-ENOMEM); err = pvr_page_table_l0_init(table, ctx->pvr_dev); if (err) { kfree(table); return ERR_PTR(err); } return table; } /** * pvr_mmu_op_context_require_sync() - Mark an MMU op context as requiring a * sync operation for the referenced page tables up to a specified level. * @op_ctx: Target MMU op context. * @level: Maximum page table level for which a sync is required. */ static void pvr_mmu_op_context_require_sync(struct pvr_mmu_op_context *op_ctx, enum pvr_mmu_sync_level level) { if (op_ctx->sync_level_required < level) op_ctx->sync_level_required = level; } /** * pvr_mmu_op_context_sync_manual() - Trigger a sync of some or all of the * page tables referenced by a MMU op context. * @op_ctx: Target MMU op context. * @level: Maximum page table level to sync. * * Do not call this function directly. Instead use * pvr_mmu_op_context_sync_partial() which is checked against the current * value of &op_ctx->sync_level_required as set by * pvr_mmu_op_context_require_sync(). */ static void pvr_mmu_op_context_sync_manual(struct pvr_mmu_op_context *op_ctx, enum pvr_mmu_sync_level level) { /* * We sync the page table levels in ascending order (starting from the * leaf node) to ensure consistency. */ WARN_ON(level < PVR_MMU_SYNC_LEVEL_NONE); if (level <= PVR_MMU_SYNC_LEVEL_NONE) return; if (op_ctx->curr_page.l0_table) pvr_page_table_l0_sync(op_ctx->curr_page.l0_table); if (level < PVR_MMU_SYNC_LEVEL_1) return; if (op_ctx->curr_page.l1_table) pvr_page_table_l1_sync(op_ctx->curr_page.l1_table); if (level < PVR_MMU_SYNC_LEVEL_2) return; pvr_page_table_l2_sync(&op_ctx->mmu_ctx->page_table_l2); } /** * pvr_mmu_op_context_sync_partial() - Trigger a sync of some or all of the * page tables referenced by a MMU op context. * @op_ctx: Target MMU op context. * @level: Requested page table level to sync up to (inclusive). * * If @level is greater than the maximum level recorded by @op_ctx as requiring * a sync operation, only the previously recorded maximum will be used. * * Additionally, if @level is greater than or equal to the maximum level * recorded by @op_ctx as requiring a sync operation, that maximum level will be * reset as a full sync will be performed. This is equivalent to calling * pvr_mmu_op_context_sync(). */ static void pvr_mmu_op_context_sync_partial(struct pvr_mmu_op_context *op_ctx, enum pvr_mmu_sync_level level) { /* * If the requested sync level is greater than or equal to the * currently required sync level, we do two things: * * Don't waste time syncing levels we haven't previously marked as * requiring a sync, and * * Reset the required sync level since we are about to sync * everything that was previously marked as requiring a sync. */ if (level >= op_ctx->sync_level_required) { level = op_ctx->sync_level_required; op_ctx->sync_level_required = PVR_MMU_SYNC_LEVEL_NONE; } pvr_mmu_op_context_sync_manual(op_ctx, level); } /** * pvr_mmu_op_context_sync() - Trigger a sync of every page table referenced by * a MMU op context. * @op_ctx: Target MMU op context. * * The maximum level marked internally as requiring a sync will be reset so * that subsequent calls to this function will be no-ops unless @op_ctx is * otherwise updated. */ static void pvr_mmu_op_context_sync(struct pvr_mmu_op_context *op_ctx) { pvr_mmu_op_context_sync_manual(op_ctx, op_ctx->sync_level_required); op_ctx->sync_level_required = PVR_MMU_SYNC_LEVEL_NONE; } /** * pvr_mmu_op_context_load_tables() - Load pointers to tables in each level of * the page table tree structure needed to reference the physical page * referenced by a MMU op context. * @op_ctx: Target MMU op context. * @should_create: Specifies whether new page tables should be created when * empty page table entries are encountered during traversal. * @load_level_required: Maximum page table level to load. * * If @should_create is %true, this function may modify the stored required * sync level of @op_ctx as new page tables are created and inserted into their * respective parents. * * Since there is only one root page table, it is technically incorrect to call * this function with a value of @load_level_required greater than or equal to * the root level number. However, this is not explicitly disallowed here. * * Return: * * 0 on success, * * Any error returned by pvr_page_table_l1_get_or_create() if * @load_level_required >= 1 except -%ENXIO, or * * Any error returned by pvr_page_table_l0_get_or_create() if * @load_level_required >= 0 except -%ENXIO. */ static int pvr_mmu_op_context_load_tables(struct pvr_mmu_op_context *op_ctx, bool should_create, enum pvr_mmu_sync_level load_level_required) { const struct pvr_page_table_l1 *l1_head_before = op_ctx->map.l1_prealloc_tables; const struct pvr_page_table_l0 *l0_head_before = op_ctx->map.l0_prealloc_tables; int err; /* Clear tables we're about to fetch in case of error states. */ if (load_level_required >= PVR_MMU_SYNC_LEVEL_1) op_ctx->curr_page.l1_table = NULL; if (load_level_required >= PVR_MMU_SYNC_LEVEL_0) op_ctx->curr_page.l0_table = NULL; /* Get or create L1 page table. */ if (load_level_required >= PVR_MMU_SYNC_LEVEL_1) { err = pvr_page_table_l1_get_or_insert(op_ctx, should_create); if (err) { /* * If @should_create is %false and no L1 page table was * found, return early but without an error. Since * pvr_page_table_l1_get_or_create() can only return * -%ENXIO if @should_create is %false, there is no * need to check it here. */ if (err == -ENXIO) err = 0; return err; } } /* Get or create L0 page table. */ if (load_level_required >= PVR_MMU_SYNC_LEVEL_0) { err = pvr_page_table_l0_get_or_insert(op_ctx, should_create); if (err) { /* * If @should_create is %false and no L0 page table was * found, return early but without an error. Since * pvr_page_table_l0_get_or_insert() can only return * -%ENXIO if @should_create is %false, there is no * need to check it here. */ if (err == -ENXIO) err = 0; /* * At this point, an L1 page table could have been * inserted but is now empty due to the failed attempt * at inserting an L0 page table. In this instance, we * must remove the empty L1 page table ourselves as * pvr_page_table_l1_remove() is never called as part * of the error path in * pvr_page_table_l0_get_or_insert(). */ if (l1_head_before != op_ctx->map.l1_prealloc_tables) { pvr_page_table_l2_remove(op_ctx); pvr_mmu_op_context_require_sync(op_ctx, PVR_MMU_SYNC_LEVEL_2); } return err; } } /* * A sync is only needed if table objects were inserted. This can be * inferred by checking if the pointer at the head of the linked list * has changed. */ if (l1_head_before != op_ctx->map.l1_prealloc_tables) pvr_mmu_op_context_require_sync(op_ctx, PVR_MMU_SYNC_LEVEL_2); else if (l0_head_before != op_ctx->map.l0_prealloc_tables) pvr_mmu_op_context_require_sync(op_ctx, PVR_MMU_SYNC_LEVEL_1); return 0; } /** * pvr_mmu_op_context_set_curr_page() - Reassign the current page of an MMU op * context, syncing any page tables previously assigned to it which are no * longer relevant. * @op_ctx: Target MMU op context. * @device_addr: New pointer target. * @should_create: Specify whether new page tables should be created when * empty page table entries are encountered during traversal. * * This function performs a full sync on the pointer, regardless of which * levels are modified. * * Return: * * 0 on success, or * * Any error returned by pvr_mmu_op_context_load_tables(). */ static int pvr_mmu_op_context_set_curr_page(struct pvr_mmu_op_context *op_ctx, u64 device_addr, bool should_create) { pvr_mmu_op_context_sync(op_ctx); op_ctx->curr_page.l2_idx = pvr_page_table_l2_idx(device_addr); op_ctx->curr_page.l1_idx = pvr_page_table_l1_idx(device_addr); op_ctx->curr_page.l0_idx = pvr_page_table_l0_idx(device_addr); op_ctx->curr_page.l1_table = NULL; op_ctx->curr_page.l0_table = NULL; return pvr_mmu_op_context_load_tables(op_ctx, should_create, PVR_MMU_SYNC_LEVEL_1); } /** * pvr_mmu_op_context_next_page() - Advance the current page of an MMU op * context. * @op_ctx: Target MMU op context. * @should_create: Specify whether new page tables should be created when * empty page table entries are encountered during traversal. * * If @should_create is %false, it is the caller's responsibility to verify that * the state of the table references in @op_ctx is valid on return. If -%ENXIO * is returned, at least one of the table references is invalid. It should be * noted that @op_ctx as a whole will be left in a valid state if -%ENXIO is * returned, unlike other error codes. The caller should check which references * are invalid by comparing them to %NULL. Only &@ptr->l2_table is guaranteed * to be valid, since it represents the root of the page table tree structure. * * Return: * * 0 on success, * * -%EPERM if the operation would wrap at the top of the page table * hierarchy, * * -%ENXIO if @should_create is %false and a page table of any level would * have otherwise been created, or * * Any error returned while attempting to create missing page tables if * @should_create is %true. */ static int pvr_mmu_op_context_next_page(struct pvr_mmu_op_context *op_ctx, bool should_create) { s8 load_level_required = PVR_MMU_SYNC_LEVEL_NONE; if (++op_ctx->curr_page.l0_idx != ROGUE_MMUCTRL_ENTRIES_PT_VALUE_X) goto load_tables; op_ctx->curr_page.l0_idx = 0; load_level_required = PVR_MMU_SYNC_LEVEL_0; if (++op_ctx->curr_page.l1_idx != ROGUE_MMUCTRL_ENTRIES_PD_VALUE) goto load_tables; op_ctx->curr_page.l1_idx = 0; load_level_required = PVR_MMU_SYNC_LEVEL_1; if (++op_ctx->curr_page.l2_idx != ROGUE_MMUCTRL_ENTRIES_PC_VALUE) goto load_tables; /* * If the pattern continued, we would set &op_ctx->curr_page.l2_idx to * zero here. However, that would wrap the top layer of the page table * hierarchy which is not a valid operation. Instead, we warn and return * an error. */ WARN(true, "%s(%p) attempted to loop the top of the page table hierarchy", __func__, op_ctx); return -EPERM; /* If indices have wrapped, we need to load new tables. */ load_tables: /* First, flush tables which will be unloaded. */ pvr_mmu_op_context_sync_partial(op_ctx, load_level_required); /* Then load tables from the required level down. */ return pvr_mmu_op_context_load_tables(op_ctx, should_create, load_level_required); } /** * DOC: Single page operations */ /** * pvr_page_create() - Create a device-virtual memory page and insert it into * a level 0 page table. * @op_ctx: Target MMU op context pointing at the device-virtual address of the * target page. * @dma_addr: DMA address of the physical page backing the created page. * @flags: Page options saved on the level 0 page table entry for reading by * the device. * * Return: * * 0 on success, or * * -%EEXIST if the requested page already exists. */ static int pvr_page_create(struct pvr_mmu_op_context *op_ctx, dma_addr_t dma_addr, struct pvr_page_flags_raw flags) { /* Do not create a new page if one already exists. */ if (pvr_page_table_l0_entry_is_valid(op_ctx->curr_page.l0_table, op_ctx->curr_page.l0_idx)) { return -EEXIST; } pvr_page_table_l0_insert(op_ctx, dma_addr, flags); pvr_mmu_op_context_require_sync(op_ctx, PVR_MMU_SYNC_LEVEL_0); return 0; } /** * pvr_page_destroy() - Destroy a device page after removing it from its * parent level 0 page table. * @op_ctx: Target MMU op context. */ static void pvr_page_destroy(struct pvr_mmu_op_context *op_ctx) { /* Do nothing if the page does not exist. */ if (!pvr_page_table_l0_entry_is_valid(op_ctx->curr_page.l0_table, op_ctx->curr_page.l0_idx)) { return; } /* Clear the parent L0 page table entry. */ pvr_page_table_l0_remove(op_ctx); pvr_mmu_op_context_require_sync(op_ctx, PVR_MMU_SYNC_LEVEL_0); } /** * pvr_mmu_op_context_destroy() - Destroy an MMU op context. * @op_ctx: Target MMU op context. */ void pvr_mmu_op_context_destroy(struct pvr_mmu_op_context *op_ctx) { const bool flush_caches = op_ctx->sync_level_required != PVR_MMU_SYNC_LEVEL_NONE; pvr_mmu_op_context_sync(op_ctx); /* Unmaps should be flushed immediately. Map flushes can be deferred. */ if (flush_caches && !op_ctx->map.sgt) pvr_mmu_flush_exec(op_ctx->mmu_ctx->pvr_dev, true); while (op_ctx->map.l0_prealloc_tables) { struct pvr_page_table_l0 *tmp = op_ctx->map.l0_prealloc_tables; op_ctx->map.l0_prealloc_tables = op_ctx->map.l0_prealloc_tables->next_free; pvr_page_table_l0_free(tmp); } while (op_ctx->map.l1_prealloc_tables) { struct pvr_page_table_l1 *tmp = op_ctx->map.l1_prealloc_tables; op_ctx->map.l1_prealloc_tables = op_ctx->map.l1_prealloc_tables->next_free; pvr_page_table_l1_free(tmp); } while (op_ctx->unmap.l0_free_tables) { struct pvr_page_table_l0 *tmp = op_ctx->unmap.l0_free_tables; op_ctx->unmap.l0_free_tables = op_ctx->unmap.l0_free_tables->next_free; pvr_page_table_l0_free(tmp); } while (op_ctx->unmap.l1_free_tables) { struct pvr_page_table_l1 *tmp = op_ctx->unmap.l1_free_tables; op_ctx->unmap.l1_free_tables = op_ctx->unmap.l1_free_tables->next_free; pvr_page_table_l1_free(tmp); } kfree(op_ctx); } /** * pvr_mmu_op_context_create() - Create an MMU op context. * @ctx: MMU context associated with owning VM context. * @sgt: Scatter gather table containing pages pinned for use by this context. * @sgt_offset: Start offset of the requested device-virtual memory mapping. * @size: Size in bytes of the requested device-virtual memory mapping. For an * unmapping, this should be zero so that no page tables are allocated. * * Returns: * * Newly created MMU op context object on success, or * * -%ENOMEM if no memory is available, * * Any error code returned by pvr_page_table_l2_init(). */ struct pvr_mmu_op_context * pvr_mmu_op_context_create(struct pvr_mmu_context *ctx, struct sg_table *sgt, u64 sgt_offset, u64 size) { int err; struct pvr_mmu_op_context *op_ctx = kzalloc(sizeof(*op_ctx), GFP_KERNEL); if (!op_ctx) return ERR_PTR(-ENOMEM); op_ctx->mmu_ctx = ctx; op_ctx->map.sgt = sgt; op_ctx->map.sgt_offset = sgt_offset; op_ctx->sync_level_required = PVR_MMU_SYNC_LEVEL_NONE; if (size) { /* * The number of page table objects we need to prealloc is * indicated by the mapping size, start offset and the sizes * of the areas mapped per PT or PD. The range calculation is * identical to that for the index into a table for a device * address, so we reuse those functions here. */ const u32 l1_start_idx = pvr_page_table_l2_idx(sgt_offset); const u32 l1_end_idx = pvr_page_table_l2_idx(sgt_offset + size); const u32 l1_count = l1_end_idx - l1_start_idx + 1; const u32 l0_start_idx = pvr_page_table_l1_idx(sgt_offset); const u32 l0_end_idx = pvr_page_table_l1_idx(sgt_offset + size); const u32 l0_count = l0_end_idx - l0_start_idx + 1; /* * Alloc and push page table entries until we have enough of * each type, ending with linked lists of l0 and l1 entries in * reverse order. */ for (int i = 0; i < l1_count; i++) { struct pvr_page_table_l1 *l1_tmp = pvr_page_table_l1_alloc(ctx); err = PTR_ERR_OR_ZERO(l1_tmp); if (err) goto err_cleanup; l1_tmp->next_free = op_ctx->map.l1_prealloc_tables; op_ctx->map.l1_prealloc_tables = l1_tmp; } for (int i = 0; i < l0_count; i++) { struct pvr_page_table_l0 *l0_tmp = pvr_page_table_l0_alloc(ctx); err = PTR_ERR_OR_ZERO(l0_tmp); if (err) goto err_cleanup; l0_tmp->next_free = op_ctx->map.l0_prealloc_tables; op_ctx->map.l0_prealloc_tables = l0_tmp; } } return op_ctx; err_cleanup: pvr_mmu_op_context_destroy(op_ctx); return ERR_PTR(err); } /** * pvr_mmu_op_context_unmap_curr_page() - Unmap pages from a memory context * starting from the current page of an MMU op context. * @op_ctx: Target MMU op context pointing at the first page to unmap. * @nr_pages: Number of pages to unmap. * * Return: * * 0 on success, or * * Any error encountered while advancing @op_ctx.curr_page with * pvr_mmu_op_context_next_page() (except -%ENXIO). */ static int pvr_mmu_op_context_unmap_curr_page(struct pvr_mmu_op_context *op_ctx, u64 nr_pages) { int err; if (nr_pages == 0) return 0; /* * Destroy first page outside loop, as it doesn't require a page * advance beforehand. If the L0 page table reference in * @op_ctx.curr_page is %NULL, there cannot be a mapped page at * @op_ctx.curr_page (so skip ahead). */ if (op_ctx->curr_page.l0_table) pvr_page_destroy(op_ctx); for (u64 page = 1; page < nr_pages; ++page) { err = pvr_mmu_op_context_next_page(op_ctx, false); /* * If the page table tree structure at @op_ctx.curr_page is * incomplete, skip ahead. We don't care about unmapping pages * that cannot exist. * * FIXME: This could be made more efficient by jumping ahead * using pvr_mmu_op_context_set_curr_page(). */ if (err == -ENXIO) continue; else if (err) return err; pvr_page_destroy(op_ctx); } return 0; } /** * pvr_mmu_unmap() - Unmap pages from a memory context. * @op_ctx: Target MMU op context. * @device_addr: First device-virtual address to unmap. * @size: Size in bytes to unmap. * * The total amount of device-virtual memory unmapped is * @nr_pages * %PVR_DEVICE_PAGE_SIZE. * * Returns: * * 0 on success, or * * Any error code returned by pvr_page_table_ptr_init(), or * * Any error code returned by pvr_page_table_ptr_unmap(). */ int pvr_mmu_unmap(struct pvr_mmu_op_context *op_ctx, u64 device_addr, u64 size) { int err = pvr_mmu_op_context_set_curr_page(op_ctx, device_addr, false); if (err) return err; return pvr_mmu_op_context_unmap_curr_page(op_ctx, size >> PVR_DEVICE_PAGE_SHIFT); } /** * pvr_mmu_map_sgl() - Map part of a scatter-gather table entry to * device-virtual memory. * @op_ctx: Target MMU op context pointing to the first page that should be * mapped. * @sgl: Target scatter-gather table entry. * @offset: Offset into @sgl to map from. Must result in a starting address * from @sgl which is CPU page-aligned. * @size: Size of the memory to be mapped in bytes. Must be a non-zero multiple * of the device page size. * @page_flags: Page options to be applied to every device-virtual memory page * in the created mapping. * * Return: * * 0 on success, * * -%EINVAL if the range specified by @offset and @size is not completely * within @sgl, or * * Any error encountered while creating a page with pvr_page_create(), or * * Any error encountered while advancing @op_ctx.curr_page with * pvr_mmu_op_context_next_page(). */ static int pvr_mmu_map_sgl(struct pvr_mmu_op_context *op_ctx, struct scatterlist *sgl, u64 offset, u64 size, struct pvr_page_flags_raw page_flags) { const unsigned int pages = size >> PVR_DEVICE_PAGE_SHIFT; dma_addr_t dma_addr = sg_dma_address(sgl) + offset; const unsigned int dma_len = sg_dma_len(sgl); struct pvr_page_table_ptr ptr_copy; unsigned int page; int err; if (size > dma_len || offset > dma_len - size) return -EINVAL; /* * Before progressing, save a copy of the start pointer so we can use * it again if we enter an error state and have to destroy pages. */ memcpy(&ptr_copy, &op_ctx->curr_page, sizeof(ptr_copy)); /* * Create first page outside loop, as it doesn't require a page advance * beforehand. */ err = pvr_page_create(op_ctx, dma_addr, page_flags); if (err) return err; for (page = 1; page < pages; ++page) { err = pvr_mmu_op_context_next_page(op_ctx, true); if (err) goto err_destroy_pages; dma_addr += PVR_DEVICE_PAGE_SIZE; err = pvr_page_create(op_ctx, dma_addr, page_flags); if (err) goto err_destroy_pages; } return 0; err_destroy_pages: memcpy(&op_ctx->curr_page, &ptr_copy, sizeof(op_ctx->curr_page)); err = pvr_mmu_op_context_unmap_curr_page(op_ctx, page); return err; } /** * pvr_mmu_map() - Map an object's virtual memory to physical memory. * @op_ctx: Target MMU op context. * @size: Size of memory to be mapped in bytes. Must be a non-zero multiple * of the device page size. * @flags: Flags from pvr_gem_object associated with the mapping. * @device_addr: Virtual device address to map to. Must be device page-aligned. * * Returns: * * 0 on success, or * * Any error code returned by pvr_page_table_ptr_init(), or * * Any error code returned by pvr_mmu_map_sgl(), or * * Any error code returned by pvr_page_table_ptr_next_page(). */ int pvr_mmu_map(struct pvr_mmu_op_context *op_ctx, u64 size, u64 flags, u64 device_addr) { struct pvr_page_table_ptr ptr_copy; struct pvr_page_flags_raw flags_raw; struct scatterlist *sgl; u64 mapped_size = 0; unsigned int count; int err; if (!size) return 0; if ((op_ctx->map.sgt_offset | size) & ~PVR_DEVICE_PAGE_MASK) return -EINVAL; err = pvr_mmu_op_context_set_curr_page(op_ctx, device_addr, true); if (err) return -EINVAL; memcpy(&ptr_copy, &op_ctx->curr_page, sizeof(ptr_copy)); flags_raw = pvr_page_flags_raw_create(false, false, flags & DRM_PVR_BO_BYPASS_DEVICE_CACHE, flags & DRM_PVR_BO_PM_FW_PROTECT); /* Map scatter gather table */ for_each_sgtable_dma_sg(op_ctx->map.sgt, sgl, count) { const size_t sgl_len = sg_dma_len(sgl); u64 sgl_offset, map_sgl_len; if (sgl_len <= op_ctx->map.sgt_offset) { op_ctx->map.sgt_offset -= sgl_len; continue; } sgl_offset = op_ctx->map.sgt_offset; map_sgl_len = min_t(u64, sgl_len - sgl_offset, size - mapped_size); err = pvr_mmu_map_sgl(op_ctx, sgl, sgl_offset, map_sgl_len, flags_raw); if (err) break; /* * Flag the L0 page table as requiring a flush when the MMU op * context is destroyed. */ pvr_mmu_op_context_require_sync(op_ctx, PVR_MMU_SYNC_LEVEL_0); op_ctx->map.sgt_offset = 0; mapped_size += map_sgl_len; if (mapped_size >= size) break; err = pvr_mmu_op_context_next_page(op_ctx, true); if (err) break; } if (err && mapped_size) { memcpy(&op_ctx->curr_page, &ptr_copy, sizeof(op_ctx->curr_page)); pvr_mmu_op_context_unmap_curr_page(op_ctx, mapped_size >> PVR_DEVICE_PAGE_SHIFT); } return err; }